bulk-release 2.20.0 → 2.21.0

package/src/main/js/processor/release.js

@@ -3,117 +3,122 @@ import {createRequire} from 'node:module'
 import {$, within} from 'zx-extra'
 import {queuefy} from 'queuefy'
 import {topo, traverseQueue} from './deps.js'
-import {createReport} from '../log.js'
+import {createReport, log} from './log.js'
 import {exec} from './exec.js'
-import {contextify, recover} from '../steps/contextify.js'
-import {fetchTags} from '../api/git.js'
-import {analyze} from '../steps/analyze.js'
-import {build} from '../steps/build.js'
-import {publish} from '../steps/publish.js'
-import {clean} from '../steps/clean.js'
-import {test} from '../steps/test.js'
+
+import {contextify} from './steps/contextify.js'
+import {recover} from './steps/teardown.js'
+import {fetchTags} from './api/git.js'
+import {analyze} from './steps/analyze.js'
+import {build} from './steps/build.js'
+import {publish} from './steps/publish.js'
+import {clean} from './steps/clean.js'
+import {test} from './steps/test.js'
+
+import meta from './publishers/meta.js'
+import npm from './publishers/npm.js'
+import ghRelease from './publishers/gh-release.js'
+import ghPages from './publishers/gh-pages.js'
+import changelog from './publishers/changelog.js'
+import cmd from './publishers/cmd.js'
+
+// Publisher registry. Order = publish order; teardown walks it in reverse.
+const publishers = [meta, npm, ghRelease, ghPages, changelog, cmd]
 
 export const run = async ({cwd = process.cwd(), env, flags = {}} = {}) => within(async () => {
+  $.memo = new Map()
+
   const {version: zbrVersion} = createRequire(import.meta.url)('../../../../package.json')
   if (flags.v || flags.version) {
     console.log(zbrVersion)
     return
   }
 
-  const context = await createContext({flags, env, cwd})
-  const {report, packages, queue, prev, graphs} = context
-  const _exec = queuefy(exec, flags.concurrency || os.cpus().length)
+  const ctx = await createContext({flags, env, cwd})
+  const {report, packages, queue, prev} = ctx
+
+  // Per-package scope: $.scope, packages[name] lookup, contextify on first touch.
+  const forEachPkg = (cb) => traverseQueue({queue, prev, cb: (name) => within(async () => {
+    $.scope = name
+    const pkg = packages[name]
+    await contextify(pkg, ctx)
+    return cb(pkg)
+  })})
 
   report
-    .log()(`zx-bulk-release@${zbrVersion}`)
-    .log()('queue:', queue)
-    .log()('graphs', graphs)
+    .log(`zx-bulk-release@${zbrVersion}`)
+    .log('queue:', queue)
+    .log('graphs', ctx.graphs)
 
   // --recover: standalone mode — clean orphan tags and exit.
-  // Run the full pipeline again after this to rebuild and publish affected packages.
   if (flags.recover) {
     await fetchTags(cwd)
     let recovered = 0
-    for (const name of queue) {
-      const pkg = packages[name]
-      await contextify(pkg, context)
-      if (await recover(pkg)) recovered++
-    }
-    report.log()(`recover: cleaned ${recovered} orphan tag(s)`)
+    await forEachPkg(async (pkg) => { if (await recover(pkg)) recovered++ })
+    report.log(`recover: cleaned ${recovered} orphan tag(s)`)
     return
   }
 
   try {
-    await traverseQueue({queue, prev, async cb(name) {
-      report.setStatus('analyzing', name)
-      const pkg = packages[name]
-      await contextify(pkg, context)
+    await forEachPkg(async (pkg) => {
+      report.setStatus('analyzing', pkg.name)
       await analyze(pkg)
-      report
-        .set('config', pkg.config, name)
-        .set('version', pkg.version, name)
-        .set('prevVersion', pkg.latest.tag?.version || pkg.manifest.version, name)
-        .set('releaseType', pkg.releaseType, name)
-        .set('tag', pkg.tag, name)
-    }})
+      report.set({
+        config:      pkg.config,
+        version:     pkg.version,
+        prevVersion: pkg.latest.tag?.version || pkg.manifest.version,
+        releaseType: pkg.releaseType,
+        tag:         pkg.tag,
+      }, pkg.name)
+    })
 
     report.setStatus('pending')
 
-    await traverseQueue({queue, prev, async cb(name) {
-      const pkg = packages[name]
-
+    await forEachPkg(async (pkg) => {
       if (!pkg.releaseType) {
-        report.setStatus('skipped', name)
+        report.setStatus('skipped', pkg.name)
         pkg.skipped = true
         return
       }
       if (flags.build !== false) {
-        report.setStatus('building', name)
-        await build(pkg, _exec)
+        report.setStatus('building', pkg.name)
+        await build(pkg)
       }
       if (flags.test !== false) {
-        report.setStatus('testing', name)
-        await test(pkg, _exec)
+        report.setStatus('testing', pkg.name)
+        await test(pkg)
       }
       if (!flags.dryRun && flags.publish !== false) {
-        report.setStatus('publishing', name)
-        await publish(pkg, _exec)
+        report.setStatus('publishing', pkg.name)
+        await publish(pkg)
       }
-
-      report.setStatus('success', name)
-    }})
+      report.setStatus('success', pkg.name)
+    })
   } catch (e) {
-    report
-      .log({level: 'error'})(e, e.stack)
-      .set('error', e)
-      .setStatus('failure')
+    report.error(e, e.stack).set('error', e).setStatus('failure')
     throw e
   } finally {
-    await clean(cwd, packages)
+    await clean(ctx)
   }
-  report
-    .setStatus('success')
-    .log()('Great success!')
+  report.setStatus('success').log('Great success!')
 })
 
 export const createContext = async ({flags, env: _env, cwd}) => {
-  const { packages, queue, root, prev, graphs } = await topo({cwd, flags})
+  const {packages, queue, root, prev, graphs} = await topo({cwd, flags})
   const report = createReport({packages, queue, flags})
   const env = {...process.env, ..._env}
 
-  $.report =        report
-  $.env =           env
-  $.verbose =       !!(flags.debug || $.env.DEBUG ) || $.verbose
-  $.quiet =         !$.verbose
+  // Register known secrets so the logger redacts them from all output.
+  log.secret(env.GH_TOKEN, env.GITHUB_TOKEN, env.NPM_TOKEN)
+
+  $.report  = report
+  $.env     = env
+  $.verbose = !!(flags.debug || env.DEBUG) || $.verbose
+  $.quiet   = !$.verbose
 
   return {
-    report,
-    packages,
-    root,
-    queue,
-    prev,
-    graphs,
-    flags,
-    env
+    cwd, env, flags, root, packages, queue, prev, graphs, report,
+    publishers,
+    run: queuefy(exec, flags.concurrency || os.cpus().length),
   }
 }

package/src/main/js/{steps → processor/steps}/analyze.js

@@ -1,15 +1,15 @@
 import {semver} from 'zx-extra'
 import {log} from '../log.js'
 import {getCommits} from '../api/git.js'
-import {updateDeps} from '../processor/deps.js'
-import {formatTag} from '../processor/meta.js';
+import {updateDeps} from '../deps.js'
+import {formatTag} from '../generators/tag.js'
 
-export const analyze = async (pkg) => {
+export const analyze = async (pkg, ctx = pkg.ctx) => {
   const semanticChanges = await getSemanticChanges(pkg.absPath, pkg.latest.tag?.ref, undefined, pkg.config.releaseRules)
   const depsChanges = await updateDeps(pkg)
   const changes = [...semanticChanges, ...depsChanges]
   const releaseType = getNextReleaseType(changes)
-  const pre = pkg.context.flags.snapshot ? `-snap.${pkg.context.git.sha.slice(0, 7)}` : undefined
+  const pre = ctx.flags.snapshot ? `-snap.${ctx.git.sha.slice(0, 7)}` : undefined
   const latestVersion = pkg.latest.tag?.version || pkg.latest.meta?.version
 
   pkg.changes = changes
@@ -24,7 +24,7 @@ export const analyze = async (pkg) => {
   pkg.manifest.version = pkg.version
   pkg.tag = releaseType ? formatTag({name: pkg.name, version: pkg.version, format: pkg.config.tagFormat}) : null
 
-  log({pkg})(
+  log.info(
     'semantic changes', changes,
     'releaseType', releaseType,
     'prevVersion', latestVersion,
@@ -46,27 +46,21 @@ export const getSemanticChanges = async (cwd, from, to, rules = semanticRules) =
   return analyzeCommits(commits, rules)
 }
 
-export const analyzeCommits = (commits, rules) =>
-  commits.reduce((acc, {subj, body, short, hash}) => {
-    rules.forEach(({group, releaseType, prefixes, keywords}) => {
-      const prefixMatcher = prefixes && new RegExp(`^(${prefixes.join('|')})(\\([a-zA-Z0-9\\-_,]+\\))?:\\s.+$`)
-      const keywordsMatcher = keywords && new RegExp(`(${keywords.join('|')}):\\s(.+)`)
-      const change = subj.match(prefixMatcher)?.[0] || body.match(keywordsMatcher)?.[2]
+export const analyzeCommits = (commits, rules) => {
+  const prepared = rules.map(({group, releaseType, prefixes, keywords}) => [
+    group,
+    releaseType,
+    prefixes && new RegExp(`^(${prefixes.join('|')})(\\([a-zA-Z0-9\\-_,]+\\))?:\\s.+$`),
+    keywords && new RegExp(`(${keywords.join('|')}):\\s(.+)`),
+  ])
 
-      if (change) {
-        acc.push({
-          group,
-          releaseType,
-          change,
-          subj,
-          body,
-          short,
-          hash
-        })
-      }
+  return commits.flatMap((commit) =>
+    prepared.flatMap(([group, releaseType, prefixRe, keywordRe]) => {
+      const change = prefixRe?.exec(commit.subj)?.[0] || keywordRe?.exec(commit.body)?.[2]
+      return change ? [{group, releaseType, change, ...commit}] : []
     })
-    return acc
-  }, [])
+  )
+}
 
 export const getNextReleaseType = (changes) => changes.length
   ? releaseSeverityOrder.find(type => changes.find(({releaseType}) => type === releaseType))

package/src/main/js/processor/steps/build.js

@@ -0,0 +1,20 @@
+import {memoizeBy} from '../../util.js'
+import {fetchPkg} from '../api/npm.js'
+import {traverseDeps} from '../deps.js'
+import {exec} from '../exec.js'
+
+export const build = memoizeBy(async (pkg, ctx = pkg.ctx) => {
+  const {run = exec, flags = {}, packages} = ctx
+  await Promise.all([
+    traverseDeps({pkg, packages, cb: ({pkg}) => build(pkg, ctx)}),
+    pkg.manifest.private !== true && pkg.changes.length === 0 && pkg.config.npmFetch && flags.npmFetch !== false
+      ? fetchPkg(pkg)
+      : Promise.resolve()
+  ])
+
+  if (!pkg.fetched) {
+    await run(pkg, 'buildCmd')
+  }
+
+  pkg.built = true
+})

package/src/main/js/processor/steps/clean.js

@@ -0,0 +1,7 @@
+import {unsetUserConfig} from '../api/git.js'
+import {npmRestore} from '../api/npm.js'
+
+export const clean = async ({cwd, packages}) => {
+  await unsetUserConfig(cwd)
+  await Promise.all(Object.values(packages).filter(p => !p.skipped).map(npmRestore))
+}

package/src/main/js/processor/steps/contextify.js

@@ -0,0 +1,49 @@
+import {getPkgConfig} from '../../config.js'
+import {getLatest} from '../generators/meta.js'
+import {getRoot, getSha} from '../api/git.js'
+
+/**
+ * Global release context — one per `run()` invocation.
+ * Built by `createContext()` in release.js and extended with runtime bits (run, publishers).
+ *
+ * @typedef {object} ReleaseContext
+ * @property {object}                 flags       CLI flags (build, test, publish, snapshot, dryRun, recover, ...).
+ * @property {Record<string,string>}  env         Resolved process env (process.env merged with per-run overrides).
+ * @property {string}                 cwd         Repo working directory.
+ * @property {object}                 root        Root package descriptor (from topo()).
+ * @property {Record<string,object>}  packages    All discovered packages keyed by name.
+ * @property {string[]}               queue       Topologically sorted release order.
+ * @property {Record<string,string[]>} prev       Per-package predecessor map.
+ * @property {object}                 graphs      Dependency graphs.
+ * @property {object}                 report      Run-wide status/report sink.
+ * @property {(pkg, name) => Promise} run         Concurrency-limited shell exec (queuefy'd).
+ * @property {object[]}               publishers  Ordered publisher registry.
+ */
+
+/**
+ * Per-package release context — refines {@link ReleaseContext} for a single package scope.
+ * Inherits all shared fields from the global ctx via prototype chain; only stores
+ * package-local state (git sha/root/tag) as own properties.
+ *
+ * @typedef {ReleaseContext & {git: {sha: string, root: string, tag?: string}}} PkgContext
+ */
+
+/**
+ * Attach per-package state to `pkg` and build its refined context.
+ * Inspired by https://docs.github.com/en/actions/learn-github-actions/contexts
+ *
+ * @param {object} pkg  Package descriptor (mutated: config, latest, context are set).
+ * @param {ReleaseContext} ctx
+ */
+export const contextify = async (pkg, ctx) => {
+  if (pkg.ctx) return
+  pkg.config = await getPkgConfig([pkg.absPath, ctx.root.absPath], ctx.env)
+  pkg.latest = await getLatest(pkg)
+  pkg.ctx = {
+    ...ctx,
+    git: {
+      sha: await getSha(pkg.absPath),
+      root: await getRoot(pkg.absPath),
+    },
+  }
+}

package/src/main/js/processor/steps/publish.js

@@ -0,0 +1,39 @@
+import {memoizeBy} from '../../util.js'
+import {exec} from '../exec.js'
+import {log} from '../log.js'
+import {npmPersist} from '../api/npm.js'
+import {pushTag} from '../api/git.js'
+import {formatTag} from '../generators/tag.js'
+import {isNpmPublished} from '../publishers/npm.js'
+import {rollbackRelease} from './teardown.js'
+
+const pushReleaseTag = async (pkg, ctx) => {
+  const {name, version, tag = formatTag({name, version}), config: {gitCommitterEmail, gitCommitterName}} = pkg
+  ctx.git.tag = tag
+  log.info(`push release tag ${tag}`)
+  await pushTag({cwd: ctx.git.root, tag, gitCommitterEmail, gitCommitterName})
+}
+
+export const publish = memoizeBy(async (pkg, ctx = pkg.ctx) => {
+  if (pkg.version !== pkg.manifest.version)
+    throw new Error('package.json version not synced')
+
+  const {run = exec, publishers = [], flags} = ctx
+  const snapshot = !!flags.snapshot
+  const active = publishers.filter(p => (!snapshot || p.snapshot) && p.when(pkg))
+
+  await npmPersist(pkg)
+
+  // Prepare phase: serial pkg mutations (e.g. meta injects into ghAssets) — must finish before any run().
+  for (const p of active) await p.prepare?.(pkg)
+
+  if (!snapshot) await pushReleaseTag(pkg, ctx)
+  try {
+    await Promise.all(active.map(p => p.run(pkg, run)))
+  } catch (e) {
+    // Roll back full release for npm-published packages; git-tag-only packages keep their tag — it IS the release.
+    if (!snapshot && isNpmPublished(pkg)) await rollbackRelease(pkg, ctx)
+    throw e
+  }
+  pkg.published = true
+})

package/src/main/js/processor/steps/teardown.js

@@ -0,0 +1,58 @@
+// Release teardown: undo a published (or half-published) release.
+//
+// Two entry points share the same core:
+//   - rollbackRelease: called inline from publish.js on mid-publish failure (tag known from pkg.ctx).
+//   - recover:         standalone --recover mode — detect orphan tags (tagged but missing on npm) and tear them down.
+//
+// Teardown walks the publishers registry in reverse and calls undo() on each that applies.
+
+import {log} from '../log.js'
+import {deleteRemoteTag} from '../api/git.js'
+import {fetchManifest} from '../api/npm.js'
+import {isNpmPublished} from '../publishers/npm.js'
+
+// Tear down a release: undo every applicable publisher, then delete the git tag.
+// Failures in individual undo steps are warned, not thrown — teardown is best-effort.
+const teardownRelease = async (pkg, ctx, {tag, version, reason}) => {
+  if (!pkg.config.ghBasicAuth) throw new Error(`${reason} requires git credentials (GH_TOKEN)`)
+
+  for (const p of [...ctx.publishers].reverse()) {
+    if (!p.undo || !p.when(pkg)) continue
+    try {
+      const result = await p.undo(pkg, {tag, version, reason})
+      if (result !== false) log.info(`${reason}: ${p.name} undone for '${tag}'`)
+    } catch (e) {
+      log.warn(`${reason}: ${p.name} undo failed`, e)
+    }
+  }
+
+  await deleteRemoteTag({cwd: ctx.git.root, tag})
+}
+
+// Rollback a release that failed mid-publish (called inline from publish.js).
+// Uses the current release tag; skips the npm existence check — we already know it failed.
+export const rollbackRelease = async (pkg, ctx = pkg.ctx) => {
+  const tag = ctx.git.tag
+  if (!tag) return
+  log.info(`rollback: cleaning up failed release for tag '${tag}'`)
+  await teardownRelease(pkg, ctx, {tag, version: pkg.version, reason: 'rollback'})
+}
+
+// Standalone recovery: if a tag exists but the package is missing from npm, treat it as an orphan and tear down.
+export const recover = async (pkg, ctx = pkg.ctx) => {
+  if (!isNpmPublished(pkg)) return false
+
+  const {tag} = pkg.latest
+  if (!tag) return false
+
+  const manifest = await fetchManifest({
+    name: pkg.name,
+    version: tag.version,
+    config: pkg.config,
+  }, {nothrow: true})
+  if (manifest) return false
+
+  log.info(`recover: tag '${tag.ref}' exists but ${pkg.name}@${tag.version} not found on npm, rolling back failed release`)
+  await teardownRelease(pkg, ctx, {tag: tag.ref, version: tag.version, reason: 'recover'})
+  return true
+}

package/src/main/js/processor/steps/test.js

@@ -0,0 +1,10 @@
+import {memoizeBy} from '../../util.js'
+import {exec} from '../exec.js'
+
+export const test = memoizeBy(async (pkg, ctx = pkg.ctx) => {
+  const {run = exec} = ctx
+  if (!pkg.fetched) {
+    await run(pkg, 'testCmd')
+  }
+  pkg.tested = true
+})

package/src/main/js/util.js

@@ -1,8 +1,4 @@
-import zlib from 'node:zlib'
-import fs from 'node:fs/promises'
-import path from 'node:path'
-import tar from 'tar-stream'
-import {Readable} from 'node:stream'
+import {$} from 'zx-extra'
 
 export const tpl = (str, context) =>
   str?.replace(/\$\{\{\s*([.a-z0-9]+)\s*}}/gi, (matched, key) => get(context, key) ?? '')
@@ -36,84 +32,43 @@ export const set = (obj, path, value) => {
 
 export const msgJoin = (rest, context, def) => tpl(rest.filter(Boolean).join(' ') || def, context)
 
-export const keyByValue = (obj, value) => Object.keys(obj).find((key) => obj[key] === value)
-
-export const memoizeBy = (fn, getKey = v => v, memo = new Map()) => async (...args) => {
-  const key = await getKey(...args)
-  if (memo.has(key)) {
-    return memo.get(key)
+// Normalize "string | {k1, k2, ...}" config shape to a positional tuple.
+// Used for shorthand configs like ghPages: 'branch from to msg' or {branch, from, to, msg}.
+export const asTuple = (opts, keys) => typeof opts === 'string'
+  ? opts.split(' ')
+  : keys.map(k => opts[k])
+
+export const attempt = async (times, action, fix) => {
+  for (let i = times; i > 0; i--) {
+    try { return await action() }
+    catch (e) {
+      if (i === 1 || !fix) throw e
+      await fix(e)
+    }
   }
-
-  const value = fn(...args)
-  memo.set(key, value)
-  return value
 }
 
-export const camelize = s => s.replace(/-./g, x => x[1].toUpperCase())
+export const attempt2 = (action, fix) => attempt(2, action, fix)
+export const attempt3 = (action, fix) => attempt(3, action, fix)
 
-export const asArray = v => Array.isArray(v) ? v : [v]
+export const memoStore = new Map()
 
-export const getCommonPath = files => {
-  const f0 = files[0]
-  const common = files.length === 1
-    ? f0.lastIndexOf('/') + 1
-    : [...(f0)].findIndex((c, i) => files.some(f => f.charAt(i) !== c))
+export const memoizeBy = (fn, getKey = v => v) => {
+  const memoized = async (...args) => {
+    const store = $.memo || memoStore
+    if (!store.has(memoized)) store.set(memoized, new Map())
+    const memo = store.get(memoized)
+    const key = await getKey(...args)
+    if (memo.has(key)) return memo.get(key)
 
-  const p = f0.slice(0, common)
-  if (p.endsWith('/')) {
-    return p
+    const value = Promise.resolve().then(() => fn(...args))
+    memo.set(key, value)
+    value.catch(() => memo.delete(key))
+    return value
   }
-
-  return p.slice(0, p.lastIndexOf('/') + 1)
+  return memoized
 }
 
-export const safePath = v => path.resolve('/', v).slice(1)
-
-// https://stackoverflow.com/questions/19978452/how-to-extract-single-file-from-tar-gz-archive-using-node-js
-export const unzip = (stream, {pick, omit, cwd = process.cwd(), strip = 0} = {}) => new Promise((resolve, reject) => {
-  const extract = tar.extract()
-  const results = []
-
-  extract.on('entry', ({name, type}, stream, cb)=>  {
-    const _name = safePath(strip ? name.split('/').slice(strip).join('/') : name)
-    const fp = path.join(cwd, _name)
-
-    let data = ''
-    stream.on('data', (chunk) => {
-      if (type !== 'file') {
-        return
-      }
-      if (omit?.includes(_name)) {
-        return
-      }
-      if (pick && !pick.includes(_name)) {
-        return
-      }
-
-      data +=chunk
-    })
-
-    stream.on('end', () => {
-      if (data) {
-        results.push(
-          fs.mkdir(path.dirname(fp), {recursive: true})
-            .then(() => fs.writeFile(fp, data, 'utf8'))
-        )
-      }
-      cb()
-    })
-
-    stream.resume()
-  })
-
-  extract.on('finish', ()=> {
-    resolve(Promise.all(results))
-  })
-
-  // fs.createReadStream('archive.tar.gz')
-  stream
-    .pipe(zlib.createGunzip())
-    .pipe(extract)
-})
-
-export const pipify = (stream) => stream.pipe ? stream : Readable.from(stream)
+export const camelize = s => s.replace(/-./g, x => x[1].toUpperCase())
+
+export const asArray = v => Array.isArray(v) ? v : [v]