@npmcli/template-oss 4.1.2 → 4.3.0

package/lib/content/release.yml

@@ -0,0 +1,125 @@
+name: Release
+
+on:
+  push:
+    branches:
+      {{#each branches}}
+      - {{ . }}
+      {{/each}}
+      {{#each releaseBranches }}
+      - {{ . }}
+      {{/each}}
+
+permissions:
+  contents: write
+  pull-requests: write
+  checks: write
+
+jobs:
+  release:
+    outputs:
+      pr: $\{{ steps.release.outputs.pr }}
+      releases: $\{{ steps.release.outputs.releases }}
+      release-flags: $\{{ steps.release.outputs.release-flags }}
+      branch: $\{{ steps.release.outputs.pr-branch }}
+      pr-number: $\{{ steps.release.outputs.pr-number }}
+      comment-id: $\{{ steps.pr-comment.outputs.result }}
+      check-id: $\{{ steps.check.outputs.check_id }}
+    {{> job jobName="Release" }}
+      - name: Release Please
+        id: release
+        env:
+          GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }}
+        run: |
+          {{ rootNpxPath }} --offline template-oss-release-please $\{{ github.ref_name }}
+      - name: Post Pull Request Comment
+        if: steps.release.outputs.pr-number
+        uses: actions/github-script@v6
+        id: pr-comment
+        env:
+          PR_NUMBER: $\{{ steps.release.outputs.pr-number }}
+        with:
+          script: |
+            const repo = { owner: context.repo.owner, repo: context.repo.repo }
+            const issue = { ...repo, issue_number: process.env.PR_NUMBER }
+
+            const { data: workflow } = await github.rest.actions.getWorkflowRun({ ...repo, run_id: context.runId })
+
+            let body = '## Release Manager\n\n'
+
+            const comments = await github.paginate(github.rest.issues.listComments, issue)
+            let commentId = comments?.find(c => c.user.login === 'github-actions[bot]' && c.body.startsWith(body))?.id
+
+            body += `- Release workflow run: ${workflow.html_url}`
+            if (commentId) {
+              await github.rest.issues.updateComment({ ...repo, comment_id: commentId, body })
+            } else {
+              const { data: comment } = await github.rest.issues.createComment({ ...issue, body })
+              commentId = comment?.id
+            }
+
+            return commentId
+      {{> stepChecks jobCheck=(obj name="Release" sha="${{ steps.release.outputs.pr-sha }}" if="steps.release.outputs.pr-number") }}
+
+  update:
+    needs: release
+    outputs:
+      sha: $\{{ steps.commit.outputs.sha }}
+      check-id: $\{{ steps.check.outputs.check_id }}
+    {{> job
+      jobName="Update - Release"
+      jobIf="needs.release.outputs.pr"
+      jobCheckout=(obj ref="${{ needs.release.outputs.branch }}" fetch-depth=0)
+    }}
+      - name: Run Post Pull Request Actions
+        env:
+          RELEASE_PR_NUMBER: $\{{ needs.release.outputs.pr-number }}
+          RELEASE_COMMENT_ID: $\{{ needs.release.outputs.comment-id }}
+          GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }}
+        run: |
+          {{ rootNpmPath }} run rp-pull-request --ignore-scripts {{ allFlags }}
+      - name: Commit
+        id: commit
+        env:
+          GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }}
+        run: |
+          git commit --all --amend --no-edit || true
+          git push --force-with-lease
+          echo "::set-output  name=sha::$(git rev-parse HEAD)"
+      {{> stepChecks jobCheck=(obj sha="${{ steps.commit.outputs.sha }}" name="Release" )}}
+      {{> stepChecks jobCheck=(obj id="${{ needs.release.outputs.check-id }}" )}}
+
+  ci:
+    name: CI - Release
+    needs: [release, update]
+    if: needs.release.outputs.pr
+    uses: ./.github/workflows/ci-release.yml
+    with:
+      ref: $\{{ needs.release.outputs.branch }}
+      check-sha: $\{{ needs.update.outputs.sha }}
+
+  post-ci:
+    needs: [release, update, ci]
+    {{> job jobName="Post CI - Release" jobIf="needs.release.outputs.pr && always()" jobSkipSetup=true }}
+      - name: Get Needs Result
+        id: needs-result
+        run: |
+          result=""
+          if [[ "$\{{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
+            result="failure"
+          elif [[ "$\{{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            result="cancelled"
+          else
+            result="success"
+          fi
+          echo "::set-output name=result::$result"
+      {{> stepChecks jobCheck=(obj id="${{ needs.update.outputs.check-id }}" status="${{ steps.needs-result.outputs.result }}") }}
+
+  post-release:
+    needs: release
+    {{> job jobName="Post Release - Release" jobIf="needs.release.outputs.releases" }}
+      - name: Run Post Release Actions
+        env:
+          RELEASES: $\{{ needs.release.outputs.releases }}
+        run: |
+          {{ rootNpmPath }} run rp-release --ignore-scripts --if-present $\{{ join(fromJSON(needs.release.outputs.release-flags), ' ') }}

package/lib/index.js

@@ -1,37 +1,42 @@
 const log = require('proc-log')
-const { defaults } = require('lodash')
+const { resolve } = require('path')
 const getConfig = require('./config.js')
 const PackageJson = require('@npmcli/package-json')
 const mapWorkspaces = require('@npmcli/map-workspaces')
 
-const getPkg = async (path, baseConfig) => {
+const getPkg = async (path) => {
   log.verbose('get-pkg', path)
 
-  const pkg = (await PackageJson.load(path)).content
-  const pkgConfig = getConfig.getPkgConfig(pkg)
+  const pkgJson = (await PackageJson.load(path)).content
+  const pkgConfig = getConfig.getPkgConfig(pkgJson)
   log.verbose('get-pkg', pkgConfig)
 
-  return { pkg, path, config: { ...baseConfig, ...pkgConfig } }
+  if (pkgConfig.content) {
+    pkgConfig.content = resolve(path, pkgConfig.content)
+  }
+
+  return {
+    pkgJson,
+    path,
+    config: pkgConfig,
+  }
 }
 
 const getWsPkgs = async (root, rootPkg) => {
   const wsPkgs = []
 
-  // workspaces are only used to filter paths and control changes to workspaces
-  // so dont pass it along with the rest of the config
-  const { workspaces, ...baseConfig } = rootPkg.config
-
   // Include all by default
+  const { workspaces } = rootPkg.config
   const include = (name) => Array.isArray(workspaces) ? workspaces.includes(name) : true
 
   // Look through all workspaces on the root pkg
-  const rootWorkspaces = await mapWorkspaces({ pkg: rootPkg.pkg, cwd: root })
+  const rootWorkspaces = await mapWorkspaces({ pkg: rootPkg.pkgJson, cwd: root })
 
   for (const [wsName, wsPath] of rootWorkspaces.entries()) {
     if (include(wsName)) {
       // A workspace can control its own workspaceRepo and workspaceModule settings
       // which are true by default on the root config
-      wsPkgs.push(await getPkg(wsPath, baseConfig))
+      wsPkgs.push(await getPkg(wsPath))
     }
   }
 
@@ -45,41 +50,33 @@ const getPkgs = async (root) => {
   log.verbose('get-pkgs', 'root', root)
 
   const rootPkg = await getPkg(root)
-  const pkgs = [rootPkg]
-
-  defaults(rootPkg.config, {
-    rootRepo: true,
-    rootModule: true,
-    workspaceRepo: true,
-    workspaceModule: true,
-    workspaces: null,
-  })
 
   const ws = await getWsPkgs(root, rootPkg)
 
   return {
-    pkgs: pkgs.concat(ws.pkgs),
+    rootPkg,
+    pkgs: [rootPkg].concat(ws.pkgs),
     workspaces: ws.paths,
   }
 }
 
-const runAll = async (root, content, checks) => {
+const runAll = async (root, checks) => {
   const results = []
-  const { pkgs, workspaces } = await getPkgs(root)
+  const { pkgs, workspaces, rootPkg: { config: rootConfig } } = await getPkgs(root)
 
-  for (const { pkg, path, config } of pkgs) {
+  for (const { pkgJson, path, config: pkgConfig } of pkgs) {
     // full config includes original config values
     const fullConfig = await getConfig({
-      pkgs,
-      workspaces,
       root,
-      pkg,
       path,
-      config,
-      content,
+      pkgJson,
+      pkgs,
+      workspaces,
+      rootConfig,
+      pkgConfig,
     })
 
-    const options = { root, pkg, path, config: fullConfig }
+    const options = { root, path, pkg: pkgJson, config: fullConfig }
     log.verbose('run-all', options)
 
     // files can export multiple checks so flatten first

package/lib/release-please/index.js

@@ -26,13 +26,34 @@ const main = async ({ repo: fullRepo, token, dryRun, branch }) => {
   )
 
   const pullRequests = await (dryRun ? manifest.buildPullRequests() : manifest.createPullRequests())
-  const releases = await (dryRun ? manifest.buildReleases() : manifest.createReleases())
+  const allReleases = await (dryRun ? manifest.buildReleases() : manifest.createReleases())
+
+  // We only ever get a single pull request with our current release-please settings
+  const rootPr = pullRequests.filter(Boolean)[0]
+  if (rootPr?.number) {
+    const commits = await github.octokit.paginate(github.octokit.rest.pulls.listCommits, {
+      owner: github.repository.owner,
+      repo: github.repository.repo,
+      pull_number: rootPr.number,
+    })
+    rootPr.sha = commits?.[commits.length - 1]?.sha
+  }
+
+  const releases = allReleases.filter(Boolean)
+  const [rootRelease, workspaceReleases] = releases.reduce((acc, r) => {
+    if (r.path === '.') {
+      acc[0] = r
+    } else {
+      acc[1].push(r)
+    }
+    return acc
+  }, [null, []])
 
   return {
-    // We only ever get a single pull request with our current release-please settings
-    pr: pullRequests.filter(Boolean)[0],
-    releases: releases.filter(Boolean),
-    release: releases.find(r => r.path === '.'),
+    pr: rootPr,
+    release: rootRelease,
+    releases: releases.length ? releases : null,
+    workspaceReleases: workspaceReleases.length ? workspaceReleases : null,
   }
 }
 

package/lib/util/files.js

@@ -1,43 +1,57 @@
 const { join } = require('path')
+const { defaultsDeep } = require('lodash')
 const { promisify } = require('util')
+const merge = require('./merge.js')
+const deepMapValues = require('just-deep-map-values')
 const glob = promisify(require('glob'))
 const Parser = require('./parser.js')
 const template = require('./template.js')
 
+const FILE_KEYS = ['rootRepo', 'rootModule', 'workspaceRepo', 'workspaceModule']
+
 const globify = pattern => pattern.split('\\').join('/')
 
-// target paths need to be joinsed with dir and templated
-const fullTarget = (dir, file, options) => join(dir, template(file, options))
+const fileEntries = (dir, files, options) => Object.entries(files)
+  // remove any false values
+  .filter(([_, v]) => v !== false)
+  // target paths need to be joinsed with dir and templated
+  .map(([k, source]) => {
+    const target = join(dir, template(k, options))
+    return [target, source]
+  })
 
 // given an obj of files, return the full target/source paths and associated parser
-const getParsers = (dir, files, options) => Object.entries(files).map(([t, s]) => {
-  let {
-    file,
-    parser: fileParser,
-    filter,
-  } = typeof s === 'string' ? { file: s } : s
-
-  file = join(options.config.sourceDir, file)
-  const target = fullTarget(dir, t, options)
-
-  if (typeof filter === 'function' && !filter(options)) {
-    return null
-  }
+const getParsers = (dir, files, options) => {
+  const parsers = fileEntries(dir, files, options).map(([target, source]) => {
+    const { file, parser, filter, clean: shouldClean } = source
+
+    if (typeof filter === 'function' && !filter(options)) {
+      return null
+    }
+
+    const clean = typeof shouldClean === 'function' ? shouldClean(options) : false
 
-  if (fileParser) {
+    if (parser) {
     // allow files to extend base parsers or create new ones
-    return new (fileParser(Parser.Parsers))(target, file, options)
-  }
+      return new (parser(Parser.Parsers))(target, file, options, { clean })
+    }
+
+    return new (Parser(file))(target, file, options, { clean })
+  })
 
-  return new (Parser(file))(target, file, options)
-})
+  return parsers.filter(Boolean)
+}
+
+const getRemovals = async (dir, files, options) => {
+  const targets = fileEntries(dir, files, options).map(([t]) => globify(t))
+  const globs = await Promise.all(targets.map(t => glob(t, { cwd: dir })))
+  return globs.flat()
+}
 
 const rmEach = async (dir, files, options, fn) => {
   const res = []
-  for (const target of files.map((t) => fullTarget(dir, t, options))) {
-    for (const file of await glob(globify(target), { cwd: dir })) {
-      res.push(await fn(file))
-    }
+  for (const file of await getRemovals(dir, files, options)) {
+    res.push(await fn(file))
   }
   return res.filter(Boolean)
 }
@@ -45,14 +59,44 @@ const rmEach = async (dir, files, options, fn) => {
 const parseEach = async (dir, files, options, fn) => {
   const res = []
   for (const parser of getParsers(dir, files, options)) {
-    if (parser) {
-      res.push(await fn(parser))
-    }
+    res.push(await fn(parser))
   }
   return res.filter(Boolean)
 }
 
+const parseConfig = (files, dir, overrides) => {
+  const normalizeFiles = (v) => deepMapValues(v, (value, key) => {
+    if (key === 'rm' && Array.isArray(value)) {
+      return value.reduce((acc, k) => {
+        acc[k] = true
+        return acc
+      }, {})
+    }
+    if (typeof value === 'string') {
+      const file = join(dir, value)
+      return key === 'file' ? file : { file }
+    }
+    if (value === true && FILE_KEYS.includes(key)) {
+      return {}
+    }
+    return value
+  })
+
+  const merged = merge(normalizeFiles(files), normalizeFiles(overrides))
+  const withDefaults = defaultsDeep(merged, FILE_KEYS.reduce((acc, k) => {
+    acc[k] = { add: {}, rm: {} }
+    return acc
+  }, {}))
+
+  return withDefaults
+}
+
+const getAddedFiles = (files) => files ? Object.keys(files.add || {}) : []
+
 module.exports = {
   rmEach,
   parseEach,
+  FILE_KEYS,
+  parseConfig,
+  getAddedFiles,
 }

package/lib/util/gitignore.js

@@ -0,0 +1,34 @@
+const { posix } = require('path')
+const { uniq } = require('lodash')
+const localeCompare = require('@isaacs/string-locale-compare')('en')
+
+const sortGitPaths = (a, b) => localeCompare(a.replace(/^!/g, ''), b.replace(/^!/g, ''))
+
+const allowDir = (p) => {
+  const parts = p.split(posix.sep)
+  return parts.flatMap((part, index, list) => {
+    const prev = list.slice(0, index)
+    const isLast = index === list.length - 1
+    const ignorePart = ['', ...prev, part, ''].join(posix.sep)
+    return [`!${ignorePart}`, !isLast && `${ignorePart}*`]
+  }).filter(Boolean)
+}
+
+const allowRootDir = (p) => {
+  // This negates the first part of each path for the gitignore
+  // files. It should be used to allow directories where everything
+  // should be allowed inside such as .github/. It shouldn't be used on
+  // directories like `workspaces/` since we want to be explicit and
+  // only allow each workspace directory individually. For those use
+  // the allowDir method above.
+  const [first, hasChildren] = p.split(posix.sep)
+  return `${first}${hasChildren ? posix.sep : ''}`
+}
+
+const gitignore = {
+  allowDir: (dirs) => uniq(dirs.map(allowDir).flat()),
+  allowRootDir: (dirs) => dirs.map(allowRootDir).map((p) => `!${posix.sep}${p}`),
+  sort: (arr) => uniq(arr.sort(sortGitPaths)),
+}
+
+module.exports = gitignore

package/lib/util/merge.js

@@ -0,0 +1,21 @@
+const { mergeWith } = require('lodash')
+
+const merge = (...objects) => mergeWith({}, ...objects, (value, srcValue, key) => {
+  if (Array.isArray(srcValue)) {
+    // Dont merge arrays, last array wins
+    return srcValue
+  }
+})
+
+const mergeWithArrays = (...keys) =>
+  (...objects) => mergeWith({}, ...objects, (value, srcValue, key) => {
+    if (Array.isArray(srcValue)) {
+      if (keys.includes(key)) {
+        return (Array.isArray(value) ? value : []).concat(srcValue)
+      }
+      return srcValue
+    }
+  })
+
+module.exports = merge
+module.exports.withArrays = mergeWithArrays

package/lib/util/parser.js

@@ -4,10 +4,13 @@ const yaml = require('yaml')
 const NpmPackageJson = require('@npmcli/package-json')
 const jsonParse = require('json-parse-even-better-errors')
 const Diff = require('diff')
-const { unset, merge } = require('lodash')
+const { unset } = require('lodash')
 const template = require('./template.js')
 const jsonDiff = require('./json-diff')
+const merge = require('./merge.js')
+
 const setFirst = (first, rest) => ({ ...first, ...rest })
+
 const traverse = (value, visit, keys = []) => {
   if (keys.length) {
     const res = visit(keys, value)
@@ -22,17 +25,25 @@ const traverse = (value, visit, keys = []) => {
   }
 }
 
+const fsOk = (code) => (error) => {
+  if (error.code === 'ENOENT') {
+    return null
+  }
+  return Object.assign(error, { code })
+}
+
 class Base {
   static types = []
-  static header = 'This file is automatically added by {{__NAME__}}. Do not edit.'
+  static header = 'This file is automatically added by {{ __NAME__ }}. Do not edit.'
   comment = (v) => v
   merge = false // supply a merge function which runs on prepare for certain types
   DELETE = template.DELETE
 
-  constructor (target, source, options) {
+  constructor (target, source, options, fileOptions) {
     this.target = target
     this.source = source
     this.options = options
+    this.fileOptions = fileOptions
   }
 
   header () {
@@ -41,6 +52,13 @@ class Base {
     }
   }
 
+  clean () {
+    if (this.fileOptions.clean) {
+      return fs.rm(this.target).catch(fsOk())
+    }
+    return null
+  }
+
   read (s) {
     return fs.readFile(s, { encoding: 'utf-8' })
   }
@@ -87,13 +105,17 @@ class Base {
   // XXX: everything is allowed to be overridden in base classes but we could
   // find a different solution than making everything public
   applyWrite () {
-    return Promise.resolve(this.read(this.source))
+    return Promise.resolve(this.clean())
+      .then(() => this.read(this.source))
       // replace template vars first, this will throw for nonexistant vars
       // because it must be parseable after this step
       .then((s) => this.template(s))
       // parse into whatever data structure is necessary for maniuplating
       // diffing, merging, etc. by default its a string
-      .then((s) => this.parse(s))
+      .then((s) => {
+        this.sourcePreParse = s
+        return this.parse(s)
+      })
       // prepare the source for writing and diffing, pass in current
       // target for merging. errors parsing or preparing targets are ok here
       .then((s) => this.applyTarget().catch(() => null).then((t) => this.prepare(s, t)))
@@ -108,14 +130,9 @@ class Base {
   }
 
   async applyDiff () {
-    const target = await this.applyTarget().catch((e) => {
-      // handle if old does not exist
-      if (e.code === 'ENOENT') {
-        return null
-      } else {
-        return { code: 'ETARGETERROR', error: e }
-      }
-    })
+    // handle if old does not exist
+    const targetError = 'ETARGETERROR'
+    const target = await this.applyTarget().catch(fsOk(targetError))
 
     // no need to diff if current file does not exist
     if (target === null) {
@@ -130,11 +147,11 @@ class Base {
 
     // if there was a target error then there is no need to diff
     // so we just show the source with an error message
-    if (target.code === 'ETARGETERROR') {
+    if (target.code === targetError) {
       const msg = `[${this.options.config.__NAME__} ERROR]`
       return [
         `${msg} There was an erroring getting the target file`,
-        `${msg} ${target.error}`,
+        `${msg} ${target}`,
         `${msg} It will be overwritten with the following source:`,
         '-'.repeat(40),
         this.toString(source),
@@ -174,7 +191,12 @@ class Yml extends Base {
   comment = (c) => ` ${c}`
 
   toString (s) {
-    return s.toString({ lineWidth: 0, indent: 2 })
+    try {
+      return s.toString({ lineWidth: 0, indent: 2 })
+    } catch (err) {
+      err.message = [this.target, this.sourcePreParse, ...s.errors, err.message].join('\n')
+      throw err
+    }
   }
 
   parse (s) {
@@ -191,6 +213,41 @@ class Yml extends Base {
   }
 }
 
+class YmlMerge extends Yml {
+  prepare (source, t) {
+    if (t === null) {
+      // If target does not exist or is in an
+      // error state, we cant do anything but write
+      // the whole document
+      return super.prepare(source)
+    }
+
+    const key = [].concat(this.key)
+
+    const getId = (node) => {
+      const index = node.items.findIndex(p => p.key?.value === this.id)
+      return index !== -1 ? node.items[index].value?.value : node.toJSON()
+    }
+
+    const target = this.parse(t)
+    const targetNodes = target.getIn(key).items.reduce((acc, node, index) => {
+      acc[getId(node)] = { node, index }
+      return acc
+    }, {})
+
+    for (const node of source.getIn(key).items) {
+      const index = targetNodes[getId(node)]?.index
+      if (typeof index === 'number' && index !== -1) {
+        target.setIn([...key, index], node)
+      } else {
+        target.addIn(key, node)
+      }
+    }
+
+    return super.prepare(target)
+  }
+}
+
 class Json extends Base {
   static types = ['json']
   // its a json comment! not really but we do add a special key
@@ -219,8 +276,8 @@ class Json extends Base {
 }
 
 class JsonMerge extends Json {
-  static header = 'This file is partially managed by {{__NAME__}}. Edits may be overwritten.'
-  merge = (t, s) => merge({}, t, s)
+  static header = 'This file is partially managed by {{ __NAME__ }}. Edits may be overwritten.'
+  merge = (t, s) => merge(t, s)
 }
 
 class PackageJson extends JsonMerge {
@@ -259,6 +316,7 @@ const Parsers = {
   Ini,
   Markdown,
   Yml,
+  YmlMerge,
   Json,
   JsonMerge,
   PackageJson,