bulk-release 2.2.13

package/src/main/js/git.js

@@ -0,0 +1,101 @@
+import {$, ctx, fs, path, tempy, copy} from 'zx-extra'
+import {log} from './log.js'
+import {memoizeBy} from './util.js'
+
+export const fetchRepo = memoizeBy(async ({cwd: _cwd, branch, origin: _origin, basicAuth}) => ctx(async ($) => {
+  const origin = _origin || (await getRepo(_cwd, {basicAuth})).repoAuthedUrl
+  const cwd = tempy.temporaryDirectory()
+  $.cwd = cwd
+  try {
+    await $`git clone --single-branch --branch ${branch} --depth 1 ${origin} .`
+  } catch (e) {
+    log({level: 'warn'})(`ref '${branch}' does not exist in ${origin}`)
+    await $`git init . &&
+            git remote add origin ${origin}`
+  }
+
+  return cwd
+}), async ({cwd, branch}) => `${await getRoot(cwd)}:${branch}`)
+
+export const pushCommit = async ({cwd, from, to, branch, origin, msg, ignoreFiles, files = [], basicAuth, gitCommitterEmail, gitCommitterName}) => ctx(async ($) => {
+  let retries = 3
+
+  const _cwd = await fetchRepo({cwd, branch, origin, basicAuth})
+  $.cwd = _cwd
+
+  for (let {relpath, contents} of files) {
+    const _contents = typeof contents === 'string' ? contents : JSON.stringify(contents, null, 2)
+    await fs.outputFile(path.resolve(_cwd, to, relpath), _contents)
+  }
+  if (from) await copy({baseFrom: cwd, from, baseTo: _cwd, to, ignoreFiles, cwd})
+
+  try {
+    await $`git config user.name ${gitCommitterName} &&
+            git config user.email ${gitCommitterEmail} &&
+            git add . &&
+            git commit -m ${msg}`
+  } catch {
+    log({level: 'warn'})(`no changes to commit to ${branch}`)
+    return
+  }
+
+  while (retries > 0) {
+    try {
+      return await $.raw`git push origin HEAD:refs/heads/${branch}`
+    } catch (e) {
+      retries -= 1
+      log({level: 'error'})('git push failed', 'branch', branch, 'retries left', retries, e)
+
+      if (retries === 0) {
+        throw e
+      }
+
+      await $`git fetch origin ${branch} &&
+              git rebase origin/${branch}`
+    }
+  }
+})
+
+export const getSha = async (cwd) => (await $.o({cwd})`git rev-parse HEAD`).toString().trim()
+
+export const getRoot = memoizeBy(async (cwd) => (await $.o({cwd})`git rev-parse --show-toplevel`).toString().trim())
+
+export const getRepo = memoizeBy(async (cwd, {basicAuth} = {}) => {
+  const originUrl = await getOrigin(cwd)
+  const [, , repoHost, repoName] = originUrl.replace(':', '/').replace(/\.git/, '').match(/.+(@|\/\/)([^/]+)\/(.+)$/) || []
+  const repoPublicUrl = `https://${repoHost}/${repoName}`
+  const repoAuthedUrl = basicAuth && repoHost && repoName
+    ? `https://${basicAuth}@${repoHost}/${repoName}.git`
+    : originUrl
+
+  return {
+    repoName,
+    repoHost,
+    repoPublicUrl,
+    repoAuthedUrl,
+    originUrl,
+  }
+}, getRoot)
+
+export const getOrigin = memoizeBy(async (cwd) =>
+  $.o({cwd})`git config --get remote.origin.url`.then(r => r.toString().trim())
+)
+
+export const getCommits = async (cwd, from, to = 'HEAD') => ctx(async ($) => {
+  const ref = from ? `${from}..${to}` : to
+
+  $.cwd = cwd
+  return (await $.raw`git log ${ref} --format=+++%s__%b__%h__%H -- .`)
+    .toString()
+    .split('+++')
+    .filter(Boolean)
+    .map(msg => {
+      const [subj, body, short, hash] = msg.split('__').map(raw => raw.trim())
+      return {subj, body, short, hash}
+    })
+})
+
+export const getTags = async (cwd, ref = '') =>
+  (await $.o({cwd})`git tag -l ${ref}`)
+    .toString()
+    .split('\n')

package/src/main/js/index.js

@@ -0,0 +1 @@
+export {run} from './processor.js'

package/src/main/js/log.js

@@ -0,0 +1,63 @@
+import {$, fs} from 'zx-extra'
+import {get, set, tpl} from './util.js'
+
+export const log = (ctx) =>
+  $.report
+    ? $.report.log(ctx)
+    : console.log
+
+export const createReport = ({logger = console, packages = {}, queue = [], flags} = {}) => ({
+  logger,
+  flags,
+  file: flags.report || flags.file,
+  status: 'initial',
+  events: [],
+  queue,
+  packages: Object.entries(packages).reduce((acc, [name, {manifest: {version}, absPath, relPath}]) => {
+    acc[name] = {
+      status: 'initial',
+      name,
+      version,
+      path: absPath,
+      relPath
+    }
+    return acc
+  }, {}),
+  get(key, pkgName) {
+    return get(
+      pkgName ? this.packages[pkgName] : this,
+      key
+    )
+  },
+  set(key, value, pkgName) {
+    set(
+      pkgName ? this.packages[pkgName] : this,
+      key,
+      value
+    )
+    return this
+  },
+  setStatus(status, name) {
+    this.set('status', status, name)
+    this.save()
+    return this
+  },
+  getStatus(status, name) {
+    return this.get('status', name)
+  },
+  log(ctx = {}) {
+    return function (...chunks) {
+      const {pkg, scope = pkg?.name || $.scope || '~', level = 'info'} = ctx
+      const msg = chunks.map(c => typeof c === 'string' ? tpl(c, ctx) : c)
+      const event = {msg, scope, date: Date.now(), level}
+      this.events.push(event)
+      logger[level](`[${scope}]`, ...msg)
+
+      return this
+    }.bind(this)
+  },
+  save() {
+    this.file && fs.outputJsonSync(this.file, this)
+    return this
+  }
+})

package/src/main/js/meta.js

@@ -0,0 +1,171 @@
+// Semantic tags processing
+
+import {Buffer} from 'node:buffer'
+import {queuefy} from 'queuefy'
+import {ctx, semver, $, fs, path} from 'zx-extra'
+import {log} from './log.js'
+import {fetchRepo, pushCommit, getTags as getGitTags} from './git.js'
+import {fetchManifest} from './npm.js'
+
+export const pushTag = (pkg) => ctx(async ($) => {
+  const {absPath: cwd, name, version, config: {gitCommitterEmail, gitCommitterName}} = pkg
+  const tag = formatTag({name, version})
+
+  pkg.context.git.tag = tag
+  log({pkg})(`push release tag ${tag}`)
+
+  $.cwd = cwd
+  await $`git config user.name ${gitCommitterName}`
+  await $`git config user.email ${gitCommitterEmail}`
+  await $`git tag -m ${tag} ${tag}`
+  await $`git push origin ${tag}`
+})
+
+export const pushMeta = queuefy(async (pkg) => {
+  log({pkg})('push artifact to branch \'meta\'')
+
+  const {name, version, absPath: cwd, config: {gitCommitterEmail, gitCommitterName, ghBasicAuth: basicAuth}} = pkg
+  const tag = formatTag({name, version})
+  const to = '.'
+  const branch = 'meta'
+  const msg = `chore: release meta ${name} ${version}`
+  const hash = (await $.o({cwd})`git rev-parse HEAD`).toString().trim()
+  const meta = {
+    META_VERSION: '1',
+    name: pkg.name,
+    hash,
+    version: pkg.version,
+    dependencies: pkg.dependencies,
+    devDependencies: pkg.devDependencies,
+    peerDependencies: pkg.peerDependencies,
+    optionalDependencies: pkg.optionalDependencies,
+  }
+  const files = [{relpath: `${getArtifactPath(tag)}.json`, contents: meta}]
+
+  await pushCommit({cwd, to, branch, msg, files, gitCommitterEmail, gitCommitterName, basicAuth})
+})
+
+export const getLatest = async (pkg) => {
+  const {absPath: cwd, name, config: {ghBasicAuth: basicAuth}} = pkg
+  const tag = await getLatestTag(cwd, name)
+  const meta = await getLatestMeta(cwd, tag?.ref, basicAuth) || await fetchManifest(pkg, {nothrow: true})
+
+  return {
+    tag,
+    meta
+  }
+}
+
+const f0 = {
+  parse(tag) {
+    if (!tag.endsWith('-f0')) return null
+
+    const pattern = /^(\d{4}\.(?:[1-9]|1[012])\.(?:[1-9]|[12]\d|30|31))-((?:[a-z0-9-]+\.)?[a-z0-9-]+)\.(v?\d+\.\d+\.\d+.*)-f0$/
+    const matched = pattern.exec(tag) || []
+    const [, _date, _name, version] = matched
+
+    if (!semver.valid(version)) return null
+
+    const date = parseDateTag(_date)
+    const name = _name.includes('.') ? `@${_name.replace('.', '/')}` : _name
+
+    return {date, name, version, format: 'f0', ref: tag}
+  },
+  format({name, date = new Date(), version}) {
+    if (!/^(@?[a-z0-9-]+\/)?[a-z0-9-]+$/.test(name) || !semver.valid(version)) return null
+
+    const d = formatDateTag(date)
+    const n = name.replace('@', '').replace('/', '.')
+
+    return `${d}-${n}.${version}-f0`
+  }
+}
+
+const f1 = {
+  parse(tag) {
+    if (!tag.endsWith('-f1')) return null
+
+    const pattern = /^(\d{4}\.(?:[1-9]|1[012])\.(?:[1-9]|[12]\d|30|31))-[a-z0-9-]+\.(v?\d+\.\d+\.\d+.*)\.([^.]+)-f1$/
+    const matched = pattern.exec(tag) || []
+    const [, _date, version, b64] = matched
+
+    if (!semver.valid(version)) return null
+
+    const date = parseDateTag(_date)
+    const name = Buffer.from(b64, 'base64url').toString('utf8')
+
+    return {date, name, version, format: 'f1', ref: tag}
+  },
+  format({name, date = new Date(), version}) {
+    if (!semver.valid(version)) return null
+
+    const b64 = Buffer.from(name).toString('base64url')
+    const d = formatDateTag(date)
+    const n = name.replace(/[^a-z0-9-]/ig, '')
+
+    return `${d}-${n}.${version}.${b64}-f1`
+  }
+}
+
+const lerna = {
+  parse(tag) {
+    const pattern = /^(@?[a-z0-9-]+(?:\/[a-z0-9-]+)?)@(v?\d+\.\d+\.\d+.*)/
+    const [, name, version] = pattern.exec(tag) || []
+
+    if (!semver.valid(version)) return null
+
+    return {name, version, format: 'lerna', ref: tag}
+  },
+  // format({name, version}) {
+  //   if (!semver.valid(version)) return null
+  //
+  //   return `${name}@${version}`
+  // }
+}
+
+// TODO
+// const variants = [f0, f1]
+// export const parseTag = (tag) => {
+//   for (const variant of variants) {
+//     const parsed = variant.parse(tag)
+//     if (parsed) return parsed
+//   }
+//
+//   return null
+// }
+
+export const parseTag = (tag) => f0.parse(tag) || f1.parse(tag) || lerna.parse(tag) || null
+
+export const formatTag = (tag) => f0.format(tag) || f1.format(tag) || null
+
+export const getTags = async (cwd, ref = '') =>
+  (await getGitTags(cwd, ref))
+    .map(tag => parseTag(tag.trim()))
+    .filter(Boolean)
+    .sort((a, b) => semver.rcompare(a.version, b.version))
+
+export const getLatestTag = async (cwd, name) =>
+  (await getTags(cwd)).find(tag => tag.name === name) || null
+
+export const getLatestTaggedVersion = async (cwd, name) =>
+  (await getLatestTag(cwd, name))?.version || null
+
+export const formatDateTag = (date = new Date()) => `${date.getUTCFullYear()}.${date.getUTCMonth() + 1}.${date.getUTCDate()}`
+
+export const parseDateTag = (date) => new Date(date.replaceAll('.', '-')+'Z')
+
+export const getArtifactPath = (tag) => tag.toLowerCase().replace(/[^a-z0-9-]/g, '-')
+
+export const getLatestMeta = async (cwd, tag, basicAuth) => {
+  if (!tag) return null
+
+  try {
+    const _cwd = await fetchRepo({cwd, branch: 'meta', basicAuth})
+    return await Promise.any([
+      fs.readJson(path.resolve(_cwd, `${getArtifactPath(tag)}.json`)),
+      fs.readJson(path.resolve(_cwd, getArtifactPath(tag), 'meta.json'))
+    ])
+  } catch {}
+
+  return null
+}

package/src/main/js/npm.js

@@ -0,0 +1,65 @@
+import {log} from './log.js'
+import {$, ctx, fs, path, INI, fetch} from 'zx-extra'
+
+export const fetchPkg = async (pkg) => {
+  const id = `${pkg.name}@${pkg.version}`
+
+  try {
+    log({pkg})(`fetching '${id}'`)
+    const cwd = pkg.absPath
+    const {npmRegistry, npmToken, npmConfig} = pkg.config
+    const bearerToken = getBearerToken(npmRegistry, npmToken, npmConfig)
+    const tarball = getTarballUrl(npmRegistry, pkg.name, pkg.version)
+    await $.raw`wget --timeout=10 --header='Authorization: ${bearerToken}' -qO- ${tarball} | tar -xvz --strip-components=1 --exclude='package.json' -C ${cwd}`
+
+    pkg.fetched = true
+  } catch (e) {
+    log({pkg, level: 'warn'})(`fetching '${id}' failed`, e)
+  }
+}
+
+export const fetchManifest = async (pkg, {nothrow} = {}) => {
+  const {npmRegistry, npmToken, npmConfig} = pkg.config
+  const bearerToken = getBearerToken(npmRegistry, npmToken, npmConfig)
+  const url = getManifestUrl(npmRegistry, pkg.name, pkg.version)
+
+  try {
+    const res = await fetch(url, {authorization: bearerToken})
+    if (!res.ok) throw res
+
+    return res.json() // NOTE .json() is async too
+  } catch (e) {
+    if (nothrow) return null
+    throw e
+  }
+}
+
+export const npmPublish = (pkg) => ctx(async ($) => {
+  const {absPath: cwd, name, version, manifest, config} = pkg
+  if (manifest.private || config?.npmPublish === false) return
+  const {npmRegistry, npmToken, npmConfig} = config
+  const npmrc = npmConfig ? npmConfig : path.resolve(cwd, '.npmrc')
+
+  log({pkg})(`publish npm package ${name} ${version} to ${npmRegistry}`)
+  $.cwd = cwd
+  if (!npmConfig) {
+    await $.raw`echo ${npmRegistry.replace(/https?:/, '')}/:_authToken=${npmToken} >> ${npmrc}`
+  }
+  await $`npm publish --no-git-tag-version --registry=${npmRegistry} --userconfig ${npmrc} --no-workspaces`
+})
+
+// $`npm view ${name}@${version} dist.tarball`
+export const getTarballUrl = (registry, name, version) => `${registry}/${name}/-/${name.replace(/^.+(%2f|\/)/,'')}-${version}.tgz`
+
+export const getManifestUrl = (registry, name, version) => `${registry}/${name}/${version}`
+
+export const getBearerToken = async (npmRegistry, npmToken, npmConfig) => {
+  const token = npmConfig
+    ? getAuthToken(npmRegistry, INI.parse(await fs.readFile(npmConfig, 'utf8')))
+    : npmToken
+  return `Bearer ${token}`
+}
+
+// NOTE registry-auth-token does not work with localhost:4873
+export const getAuthToken = (registry, npmrc) =>
+  (Object.entries(npmrc).find(([reg]) => reg.startsWith(registry.replace(/^https?/, ''))) || [])[1]

package/src/main/js/processor.js

@@ -0,0 +1,155 @@
+import os from 'node:os'
+import {$, fs, within} from 'zx-extra'
+import {queuefy} from 'queuefy'
+import {analyze} from './analyze.js'
+import {pushChangelog} from './changelog.js'
+import {getPkgConfig} from './config.js'
+import {topo, traverseDeps, traverseQueue} from './deps.js'
+import {ghPages, ghRelease} from './gh.js'
+import {getRoot, getSha} from './git.js'
+import {log, createReport} from './log.js'
+import {getLatest, pushMeta, pushTag} from './meta.js'
+import {fetchPkg, npmPublish} from './npm.js'
+import {memoizeBy, tpl} from './util.js'
+
+export const run = async ({cwd = process.cwd(), env, flags = {}} = {}) => within(async () => {
+  const context = await createContext({flags, env, cwd})
+  const {report, packages, queue, prev, graphs} = context
+  const _runCmd = queuefy(runCmd, flags.concurrency || os.cpus().length)
+
+  report
+    .log()('zx-bulk-release')
+    .log()('queue:', queue)
+    .log()('graphs', graphs)
+
+  try {
+    await traverseQueue({queue, prev, async cb(name) {
+      report.setStatus('analyzing', name)
+      const pkg = packages[name]
+      await contextify(pkg, context)
+      await analyze(pkg)
+      report
+        .set('config', pkg.config, name)
+        .set('version', pkg.version, name)
+        .set('prevVersion', pkg.latest.tag?.version || pkg.manifest.version, name)
+        .set('releaseType', pkg.releaseType, name)
+        .set('tag', pkg.tag, name)
+    }})
+
+    report.setStatus('pending')
+
+    await traverseQueue({queue, prev, async cb(name) {
+      const pkg = packages[name]
+
+      if (!pkg.releaseType) {
+        report.setStatus('skipped', name)
+        return
+      }
+      if (!flags.noBuild) {
+        report.setStatus('building', name)
+        await build(pkg, _runCmd)
+      }
+      if (!flags.dryRun && !flags.noPublish) {
+        report.setStatus('publishing', name)
+        await publish(pkg, _runCmd)
+      }
+
+      report.setStatus('success', name)
+    }})
+  } catch (e) {
+    report
+      .log({level: 'error'})(e, e.stack)
+      .set('error', e)
+      .setStatus('failure')
+    throw e
+  }
+  report
+    .setStatus('success')
+    .log()('Great success!')
+})
+
+export const runCmd = async (pkg, name) => {
+  const cmd = tpl(pkg.config[name], {...pkg, ...pkg.context})
+
+  if (cmd) {
+    log({pkg})(`run ${name} '${cmd}'`)
+    return $.o({cwd: pkg.absPath, quote: v => v, preferLocal: true})`${cmd}`
+  }
+}
+
+const createContext = async ({flags, env, cwd}) => {
+  const { packages, queue, root, prev, graphs } = await topo({cwd, flags})
+  const report =    createReport({packages, queue, flags})
+
+  $.report =        report
+  $.env =           {...process.env, ...env}
+  $.verbose =       !!(flags.debug || $.env.DEBUG ) || $.verbose
+
+  return {
+    report,
+    packages,
+    root,
+    queue,
+    prev,
+    graphs,
+    flags
+  }
+}
+
+// Inspired by https://docs.github.com/en/actions/learn-github-actions/contexts
+const contextify = async (pkg, {packages, root}) => {
+  pkg.config = await getPkgConfig(pkg.absPath, root.absPath)
+  pkg.latest = await getLatest(pkg)
+  pkg.context = {
+    git: {
+      sha: await getSha(pkg.absPath),
+      root: await getRoot(pkg.absPath)
+    },
+    env: $.env,
+    packages
+  }
+}
+
+const build = memoizeBy(async (pkg, run = runCmd, flags = {}, self = build) => within(async () => {
+  $.scope = pkg.name
+
+  await Promise.all([
+    traverseDeps(pkg, pkg.context.packages, async (_, {pkg}) => self(pkg, run, flags, self)),
+    pkg.changes.length === 0 && pkg.config.npmFetch && !flags.noNpmFetch
+      ? fetchPkg(pkg)
+      : Promise.resolve()
+  ])
+
+  if (!pkg.fetched) {
+    await run(pkg, 'buildCmd')
+    await run(pkg, 'testCmd')
+  }
+
+  pkg.built = true
+}))
+
+const publish = memoizeBy(async (pkg, run = runCmd) => within(async () => {
+  $.scope = pkg.name
+
+  // Debug
+  // https://www.npmjs.com/package/@packasso/preset-ts-tsc-uvu/v/0.0.0?activeTab=code
+  // https://github.com/qiwi/packasso/actions/runs/4514909191/jobs/7951564982#step:7:817
+  // https://github.com/qiwi/packasso/blob/meta/2023-3-24-packasso-preset-ts-tsc-uvu-0-21-0-f0.json
+  if (pkg.version !== pkg.manifest.version) {
+    throw new Error('package.json version not synced')
+  }
+
+  fs.writeJsonSync(pkg.manifestPath, pkg.manifest, {spaces: 2})
+  await pushTag(pkg)
+
+  await Promise.all([
+    pushMeta(pkg),
+    pushChangelog(pkg),
+    npmPublish(pkg),
+    ghRelease(pkg),
+    ghPages(pkg),
+    run(pkg, 'publishCmd')
+  ])
+
+  pkg.published = true
+}))

package/src/main/js/util.js

@@ -0,0 +1,46 @@
+export const tpl = (str, context) =>
+  str?.replace(/\$\{\{\s*([.a-z0-9]+)\s*}}/gi, (matched, key) => get(context, key) ?? '')
+
+export const get = (obj, path = '.') => {
+    const chunks = path.split('.').filter(Boolean)
+    let result = obj
+
+    for (let i = 0, len = chunks.length; i < len && result !== undefined && result !== null; i++) {
+        result = result[chunks[i]]
+    }
+
+    return result
+}
+
+export const set = (obj, path, value) => {
+    const chunks = path.split('.').filter(Boolean)
+    let result = obj
+
+    for (let i = 0, len = chunks.length; i < len && result !== undefined && result !== null; i++) {
+        if (i === len - 1) {
+            result[chunks[i]] = value
+        } else {
+            result[chunks[i]] = result[chunks[i]] || {}
+            result = result[chunks[i]]
+        }
+    }
+
+    return result
+}
+
+export const msgJoin = (rest, context, def) => tpl(rest.filter(Boolean).join(' ') || def, context)
+
+export const keyByValue = (obj, value) => Object.keys(obj).find((key) => obj[key] === value)
+
+export const memoizeBy = (fn, getKey = v => v, memo = new Map()) => async (...args) => {
+  const key = await getKey(...args)
+  if (memo.has(key)) {
+    return memo.get(key)
+  }
+
+  const value = fn(...args)
+  memo.set(key, value)
+  return value
+}
+
+export const camelize = s => s.replace(/-./g, x => x[1].toUpperCase())

package/src/test/js/test-utils.js

@@ -0,0 +1,63 @@
+import {ctx, fs, path, tempy, $, sleep} from 'zx-extra'
+import {fileURLToPath} from 'node:url'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+export const fixtures = path.resolve(__dirname, '../fixtures')
+
+export const createNpmRegistry = () => {
+  let p
+
+  return {
+    address: $.env.NPM_REGISTRY,
+    async start() {
+      fs.removeSync(path.resolve(__dirname, '../../../storage'))
+      const config = path.resolve(__dirname, '../../../verdaccio.config.yaml')
+      ctx(($) => {
+        $.preferLocal = true
+        p = $`verdaccio --config ${config}`
+      })
+
+      return sleep(1000)
+    },
+    async stop() {
+      p._nothrow = true
+      return p?.kill()
+    }
+  }
+}
+
+export const createFakeRepo = async ({cwd = tempy.temporaryDirectory(), commits = []} = {}) =>
+  ctx(async ($) => {
+    $.cwd = cwd
+    await $`git init`
+
+    await addCommits({cwd, commits})
+
+    const bare = tempy.temporaryDirectory()
+    await $`git init --bare ${bare}`
+    await $`git remote add origin ${bare}`
+
+    return cwd
+  })
+
+export const addCommits = async ({cwd, commits = []}) => ctx(async ($) => {
+  $.cwd = cwd
+
+  for (let {msg, files, name = 'Semrel-extra Bot', email = 'semrel-extra-bot@hotmail.com', tags = []} of commits) {
+    await $`git config user.name ${name}`
+    await $`git config user.email ${email}`
+    for (let {relpath, contents} of files) {
+      const _contents = typeof contents === 'string' ? contents : JSON.stringify(contents, null, 2)
+      const file = path.resolve(cwd, relpath)
+      await fs.outputFile(file, _contents)
+
+      await $`git add ${file}`
+    }
+
+    await $`git commit -m ${msg}`
+
+    for (let tag of tags) {
+      await $`git tag ${tag} -m ${tag}`
+    }
+  }
+})