@npmcli/template-oss 4.1.2 → 4.2.0

package/lib/content/index.js

@@ -5,9 +5,9 @@ const releasePlease = () => ({
     file: 'release-please.yml',
     filter: (o) => !o.pkg.private,
   },
-  '.github/workflows/release-test.yml': {
-    file: 'release-test.yml',
-    filter: (o) => !o.pkg.private && o.config.releaseTest === 'release-test.yml',
+  '.github/workflows/release.yml': {
+    file: 'release.yml',
+    filter: (o) => !o.pkg.private,
   },
   '.release-please-manifest.json': {
     file: 'release-please-manifest.json',
@@ -25,22 +25,29 @@ const releasePlease = () => ({
   },
 })
 
+const tap = (name) => ({
+  '.github/matchers/tap.json': 'tap.json',
+  [`.github/workflows/ci${name ? `-${name}` : ''}.yml`]: 'ci.yml',
+})
+
 // Changes applied to the root of the repo
 const rootRepo = {
   add: {
     '.commitlintrc.js': 'commitlintrc.js',
-    '.github/workflows/ci.yml': 'ci.yml',
     '.github/ISSUE_TEMPLATE/bug.yml': 'bug.yml',
     '.github/ISSUE_TEMPLATE/config.yml': 'config.yml',
     '.github/CODEOWNERS': 'CODEOWNERS',
     '.github/dependabot.yml': 'dependabot.yml',
-    '.github/matchers/tap.json': 'tap.json',
     '.github/workflows/audit.yml': 'audit.yml',
     '.github/workflows/codeql-analysis.yml': 'codeql-analysis.yml',
     '.github/workflows/post-dependabot.yml': 'post-dependabot.yml',
     '.github/workflows/pull-request.yml': 'pull-request.yml',
     ...releasePlease(),
+    ...tap(),
   },
+  rm: [
+    '.github/workflows/release-test.yml',
+  ],
 }
 
 // These are also applied to the root of the repo
@@ -65,9 +72,8 @@ const rootModule = {
 // Changes for each workspace but applied to the root of the repo
 const workspaceRepo = {
   add: {
-    ...releasePlease(true),
-    '.github/matchers/tap.json': 'tap.json',
-    '.github/workflows/ci-{{pkgNameFs}}.yml': 'ci.yml',
+    ...releasePlease(),
+    ...tap('{{pkgNameFs}}'),
   },
   rm: [
     // These are the old release please files that should be removed now
@@ -95,17 +101,33 @@ module.exports = {
   workspaceRepo,
   workspaceModule,
   windowsCI: true,
+  macCI: true,
   branches: ['main', 'latest'],
   releaseBranches: [],
   defaultBranch: 'main',
-  // Escape hatch since we write a release test file but the
-  // CLI has a very custom one we dont want to overwrite. This
-  // setting allows us to call a workflow by any name during release
-  releaseTest: 'release-test.yml',
-  distPaths: ['bin/', 'lib/'],
+  distPaths: [
+    'bin/',
+    'lib/',
+  ],
+  allowPaths: [
+    '/bin/',
+    '/lib/',
+    '/.eslintrc.local.*',
+    '**/.gitignore',
+    '/docs/',
+    '/tap-snapshots/',
+    '/test/',
+    '/map.js',
+    '/scripts/',
+    '/README*',
+    '/LICENSE*',
+    '/CHANGELOG*',
+  ],
+  ignorePaths: [],
   ciVersions: ['14.17.0', '14.x', '16.13.0', '16.x', '18.0.0', '18.x'],
   lockfile: false,
-  npmBin: 'npm',
+  npm: 'npm',
+  npx: 'npx',
   unwantedPackages: [
     'eslint',
     'eslint-plugin-node',

package/lib/content/pkg.json

@@ -5,23 +5,41 @@
     "lint": "eslint \"**/*.js\"",
     "postlint": "template-oss-check",
     "template-oss-apply": "template-oss-apply --force",
-    "lintfix": "{{npmBin}} run lint -- --fix",
+    "lintfix": "{{localNpmPath}} run lint -- --fix",
     "preversion": {{{del}}},
     "postversion": {{{del}}},
     "prepublishOnly": {{{del}}},
+    "postpublish": {{{del}}},
     "snap": "tap",
     "test": "tap",
-    "posttest": "{{npmBin}} run lint",
+    "posttest": "{{localNpmPath}} run lint",
+    {{#if isRootMono}}
+    "test-all": "{{localNpmPath}} run test -ws -iwr --if-present",
+    "lint-all": "{{localNpmPath}} run lint -ws -iwr --if-present",
+    {{/if}}
     "template-copy": {{{del}}},
     "lint:fix": {{{del}}}
   },
   "repository": {{#if repository}}{{{json repository}}}{{else}}{{{del}}}{{/if}},
   "engines": {
+    {{#if engines}}
     "node": {{{json engines}}}
+    {{/if}}
   },
   {{{json __CONFIG_KEY__}}}: {
     "version": {{#if isDogFood}}{{{del}}}{{else}}{{{json __VERSION__}}}{{/if}}
   },
   "templateVersion": {{{del}}},
-  "standard": {{{del}}}
+  "standard": {{{del}}},
+  "tap": {
+    {{#if isRootMono}}
+    "test-ignore": "^({{#each workspaceGlobs}}{{this}}{{#unless @last}}|{{/unless}}{{/each}})/",
+    {{/if}}
+    "nyc-arg": [
+      {{#if isRootMono}}
+      {{#each workspaceGlobs}}"--exclude", "{{this}}/**",{{/each}}
+      {{/if}}
+      "--exclude", "tap-snapshots/**"
+    ]
+  }
 }

package/lib/content/post-dependabot.yml

@@ -9,12 +9,7 @@ permissions:
 
 jobs:
   template-oss-apply:
-    runs-on: ubuntu-latest
-    if: github.actor == 'dependabot[bot]'
-    steps:
-      {{> setupGit with=(obj ref="${{ github.event.pull_request.head_ref }}")}}
-      {{> setupNode}}
-      {{> setupDeps}}
+    {{> setupJob jobIf="github.actor == 'dependabot[bot]'" checkout=(obj ref="${{ github.event.pull_request.head_ref }}")}}
       - name: Dependabot metadata
         id: metadata
         uses: dependabot/fetch-metadata@v1.1.1
@@ -25,7 +20,7 @@ jobs:
         env:
           GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }}
         run: |
-          npm run template-oss-apply
+          {{rootNpmPath}} run template-oss-apply
           git commit -am "chore: postinstall for dependabot template-oss PR"
           git push
-          npm run lint
+          {{rootNpmPath}} run lint

package/lib/content/pull-request.yml

@@ -11,14 +11,10 @@ on:
 jobs:
   check:
     name: Check PR Title or Commits
-    runs-on: ubuntu-latest
-    steps:
-      {{> setupGit with=(obj fetch-depth=0)}}
-      {{> setupNode}}
-      {{> setupDeps}}
+    {{> setupJob checkout=(obj fetch-depth=0)}}
       - name: Check commits or PR title
         env:
           PR_TITLE: $\{{ github.event.pull_request.title }}
         run: |
-          npx --offline commitlint -V --from origin/{{defaultBranch}} --to $\{{ github.event.pull_request.head.sha }} \
+          {{rootNpxPath}} --offline commitlint -V --from origin/{{defaultBranch}} --to $\{{ github.event.pull_request.head.sha }} \
             || echo $PR_TITLE | npx --offline commitlint -V

package/lib/content/release-please.yml

@@ -16,14 +16,10 @@ permissions:
 
 jobs:
   release-please:
-    runs-on: ubuntu-latest
     outputs:
       pr: $\{{ steps.release.outputs.pr }}
       release: $\{{ steps.release.outputs.release }}
-    steps:
-      {{> setupGit}}
-      {{> setupNode}}
-      {{> setupDeps}}
+    {{> setupJob }}
       - name: Release Please
         id: release
         run: npx --offline template-oss-release-please $\{{ github.ref_name }}
@@ -40,34 +36,29 @@ jobs:
       - name: Output ref
         id: ref
         run: echo "::set-output name=branch::$\{{ fromJSON(needs.release-please.outputs.pr).headBranchName }}"
-      {{> setupGit with=(obj ref="${{ steps.ref.outputs.branch }}" fetch-depth=0)}}
+      {{> setupGit checkout=(obj ref="${{ steps.ref.outputs.branch }}" fetch-depth=0)}}
       {{> setupNode}}
       {{> setupDeps}}
       - name: Post pull request actions
         env:
           GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }}
         run: |
-          npm run rp-pull-request --ignore-scripts --if-present -ws -iwr
+          {{rootNpmPath}} run rp-pull-request --ignore-scripts --if-present -ws -iwr
           git commit -am "chore: post pull request" || true
           git push
   
   release-test:
     needs: post-pr
     if: needs.post-pr.outputs.ref
-    uses: ./.github/workflows/{{releaseTest}}
+    uses: ./.github/workflows/release.yml
     with:
       ref: $\{{ needs.post-pr.outputs.ref }}
 
   post-release:
     needs: release-please
-    if: needs.release-please.outputs.release
-    runs-on: ubuntu-latest
-    steps:
-      {{> setupGit}}
-      {{> setupNode}}
-      {{> setupDeps}}
+    {{> setupJob jobIf="needs.release-please.outputs.release" }}
       - name: Post release actions
         env:
           GITHUB_TOKEN: $\{{ secrets.GITHUB_TOKEN }}
         run: |
-          npm run rp-release --ignore-scripts --if-present -ws -iwr
+          {{rootNpmPath}} run rp-release --ignore-scripts --if-present -ws -iwr

package/lib/content/release.yml

@@ -0,0 +1,20 @@
+
+name: Release
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        required: true
+        type: string
+
+jobs:
+  lint-all:
+    {{> setupJob checkout=(obj ref="${{ inputs.ref }}")}}
+      - run: {{rootNpmPath}} run lint -ws -iwr --if-present
+
+  test-all:
+    {{> setupJobMatrix checkout=(obj ref="${{ inputs.ref }}")}}
+      - name: add tap problem matcher
+        run: echo "::add-matcher::.github/matchers/tap.json"
+      - run: {{rootNpmPath}} run test -ws -iwr --if-present

package/lib/index.js

@@ -1,17 +1,25 @@
 const log = require('proc-log')
-const { defaults } = require('lodash')
+const { resolve } = require('path')
 const getConfig = require('./config.js')
 const PackageJson = require('@npmcli/package-json')
 const mapWorkspaces = require('@npmcli/map-workspaces')
 
-const getPkg = async (path, baseConfig) => {
+const getPkg = async (path) => {
   log.verbose('get-pkg', path)
 
   const pkg = (await PackageJson.load(path)).content
   const pkgConfig = getConfig.getPkgConfig(pkg)
   log.verbose('get-pkg', pkgConfig)
 
-  return { pkg, path, config: { ...baseConfig, ...pkgConfig } }
+  if (pkgConfig.content) {
+    pkgConfig.content = resolve(path, pkgConfig.content)
+  }
+
+  return {
+    pkg,
+    path,
+    config: pkgConfig,
+  }
 }
 
 const getWsPkgs = async (root, rootPkg) => {
@@ -45,38 +53,30 @@ const getPkgs = async (root) => {
   log.verbose('get-pkgs', 'root', root)
 
   const rootPkg = await getPkg(root)
-  const pkgs = [rootPkg]
-
-  defaults(rootPkg.config, {
-    rootRepo: true,
-    rootModule: true,
-    workspaceRepo: true,
-    workspaceModule: true,
-    workspaces: null,
-  })
 
   const ws = await getWsPkgs(root, rootPkg)
 
   return {
-    pkgs: pkgs.concat(ws.pkgs),
+    rootPkg,
+    pkgs: [rootPkg].concat(ws.pkgs),
     workspaces: ws.paths,
   }
 }
 
-const runAll = async (root, content, checks) => {
+const runAll = async (root, checks) => {
   const results = []
-  const { pkgs, workspaces } = await getPkgs(root)
+  const { pkgs, workspaces, rootPkg: { config: rootConfig } } = await getPkgs(root)
 
-  for (const { pkg, path, config } of pkgs) {
+  for (const { pkg, path, config: pkgConfig } of pkgs) {
     // full config includes original config values
     const fullConfig = await getConfig({
-      pkgs,
-      workspaces,
       root,
-      pkg,
       path,
-      config,
-      content,
+      pkg,
+      pkgs,
+      workspaces,
+      rootConfig,
+      pkgConfig,
     })
 
     const options = { root, pkg, path, config: fullConfig }

package/lib/util/files.js

@@ -1,43 +1,55 @@
 const { join } = require('path')
+const { defaultsDeep } = require('lodash')
 const { promisify } = require('util')
+const merge = require('./merge.js')
+const deepMapValues = require('just-deep-map-values')
 const glob = promisify(require('glob'))
 const Parser = require('./parser.js')
 const template = require('./template.js')
 
+const FILE_KEYS = ['rootRepo', 'rootModule', 'workspaceRepo', 'workspaceModule']
+
 const globify = pattern => pattern.split('\\').join('/')
 
-// target paths need to be joinsed with dir and templated
-const fullTarget = (dir, file, options) => join(dir, template(file, options))
+const fileEntries = (dir, files, options) => Object.entries(files)
+  // remove any false values
+  .filter(([_, v]) => v !== false)
+  // target paths need to be joinsed with dir and templated
+  .map(([k, source]) => {
+    const target = join(dir, template(k, options))
+    return [target, source]
+  })
 
 // given an obj of files, return the full target/source paths and associated parser
-const getParsers = (dir, files, options) => Object.entries(files).map(([t, s]) => {
-  let {
-    file,
-    parser: fileParser,
-    filter,
-  } = typeof s === 'string' ? { file: s } : s
-
-  file = join(options.config.sourceDir, file)
-  const target = fullTarget(dir, t, options)
-
-  if (typeof filter === 'function' && !filter(options)) {
-    return null
-  }
+const getParsers = (dir, files, options) => {
+  const parsers = fileEntries(dir, files, options).map(([target, source]) => {
+    const { file, parser, filter } = source
+
+    if (typeof filter === 'function' && !filter(options)) {
+      return null
+    }
 
-  if (fileParser) {
+    if (parser) {
     // allow files to extend base parsers or create new ones
-    return new (fileParser(Parser.Parsers))(target, file, options)
-  }
+      return new (parser(Parser.Parsers))(target, file, options)
+    }
+
+    return new (Parser(file))(target, file, options)
+  })
 
-  return new (Parser(file))(target, file, options)
-})
+  return parsers.filter(Boolean)
+}
+
+const getRemovals = async (dir, files, options) => {
+  const targets = fileEntries(dir, files, options).map(([t]) => globify(t))
+  const globs = await Promise.all(targets.map(t => glob(t, { cwd: dir })))
+  return globs.flat()
+}
 
 const rmEach = async (dir, files, options, fn) => {
   const res = []
-  for (const target of files.map((t) => fullTarget(dir, t, options))) {
-    for (const file of await glob(globify(target), { cwd: dir })) {
-      res.push(await fn(file))
-    }
+  for (const file of await getRemovals(dir, files, options)) {
+    res.push(await fn(file))
   }
   return res.filter(Boolean)
 }
@@ -45,14 +57,44 @@ const rmEach = async (dir, files, options, fn) => {
 const parseEach = async (dir, files, options, fn) => {
   const res = []
   for (const parser of getParsers(dir, files, options)) {
-    if (parser) {
-      res.push(await fn(parser))
-    }
+    res.push(await fn(parser))
   }
   return res.filter(Boolean)
 }
 
+const parseConfig = (files, dir, overrides) => {
+  const normalizeFiles = (v) => deepMapValues(v, (value, key) => {
+    if (key === 'rm' && Array.isArray(value)) {
+      return value.reduce((acc, k) => {
+        acc[k] = true
+        return acc
+      }, {})
+    }
+    if (typeof value === 'string') {
+      const file = join(dir, value)
+      return key === 'file' ? file : { file }
+    }
+    if (value === true && FILE_KEYS.includes(key)) {
+      return {}
+    }
+    return value
+  })
+
+  const merged = merge(normalizeFiles(files), normalizeFiles(overrides))
+  const withDefaults = defaultsDeep(merged, FILE_KEYS.reduce((acc, k) => {
+    acc[k] = { add: {}, rm: {} }
+    return acc
+  }, {}))
+
+  return withDefaults
+}
+
+const getAddedFiles = (files) => files ? Object.keys(files.add || {}) : []
+
 module.exports = {
   rmEach,
   parseEach,
+  FILE_KEYS,
+  parseConfig,
+  getAddedFiles,
 }

package/lib/util/gitignore.js

@@ -0,0 +1,34 @@
+const { posix } = require('path')
+const { uniq } = require('lodash')
+const localeCompare = require('@isaacs/string-locale-compare')('en')
+
+const sortGitPaths = (a, b) => localeCompare(a.replace(/^!/g, ''), b.replace(/^!/g, ''))
+
+const allowDir = (p) => {
+  const parts = p.split(posix.sep)
+  return parts.flatMap((part, index, list) => {
+    const prev = list.slice(0, index)
+    const isLast = index === list.length - 1
+    const ignorePart = ['', ...prev, part, ''].join(posix.sep)
+    return [`!${ignorePart}`, !isLast && `${ignorePart}*`]
+  }).filter(Boolean)
+}
+
+const allowRootDir = (p) => {
+  // This negates the first part of each path for the gitignore
+  // files. It should be used to allow directories where everything
+  // should be allowed inside such as .github/. It shouldn't be used on
+  // directories like `workspaces/` since we want to be explicit and
+  // only allow each workspace directory individually. For those use
+  // the allowDir method above.
+  const [first, hasChildren] = p.split(posix.sep)
+  return `${first}${hasChildren ? posix.sep : ''}`
+}
+
+const gitignore = {
+  allowDir: (dirs) => uniq(dirs.map(allowDir).flat()),
+  allowRootDir: (dirs) => dirs.map(allowRootDir).map((p) => `!${posix.sep}${p}`),
+  sort: (arr) => uniq(arr.sort(sortGitPaths)),
+}
+
+module.exports = gitignore

package/lib/util/merge.js

@@ -0,0 +1,21 @@
+const { mergeWith } = require('lodash')
+
+const merge = (...objects) => mergeWith({}, ...objects, (value, srcValue, key) => {
+  if (Array.isArray(srcValue)) {
+    // Dont merge arrays, last array wins
+    return srcValue
+  }
+})
+
+const mergeWithArrays = (...keys) =>
+  (...objects) => mergeWith({}, ...objects, (value, srcValue, key) => {
+    if (Array.isArray(srcValue)) {
+      if (keys.includes(key)) {
+        return (Array.isArray(value) ? value : []).concat(srcValue)
+      }
+      return srcValue
+    }
+  })
+
+module.exports = merge
+module.exports.withArrays = mergeWithArrays

package/lib/util/parser.js

@@ -4,9 +4,10 @@ const yaml = require('yaml')
 const NpmPackageJson = require('@npmcli/package-json')
 const jsonParse = require('json-parse-even-better-errors')
 const Diff = require('diff')
-const { unset, merge } = require('lodash')
+const { unset } = require('lodash')
 const template = require('./template.js')
 const jsonDiff = require('./json-diff')
+const merge = require('./merge.js')
 const setFirst = (first, rest) => ({ ...first, ...rest })
 const traverse = (value, visit, keys = []) => {
   if (keys.length) {
@@ -220,7 +221,7 @@ class Json extends Base {
 
 class JsonMerge extends Json {
   static header = 'This file is partially managed by {{__NAME__}}. Edits may be overwritten.'
-  merge = (t, s) => merge({}, t, s)
+  merge = (t, s) => merge(t, s)
 }
 
 class PackageJson extends JsonMerge {

package/lib/util/template.js

@@ -3,35 +3,37 @@ const { basename, extname, join } = require('path')
 const fs = require('fs')
 const DELETE = '__DELETE__'
 
-const partialName = (s) =>
-  basename(s, extname(s)).replace(/-([a-z])/g, (_, g) => g.toUpperCase())
+const partialName = (s) => basename(s, extname(s)) // remove extension
+  .replace(/^_/, '') // remove leading underscore
+  .replace(/-([a-z])/g, (_, g) => g.toUpperCase()) // camelcase
 
-const setupHandlebars = (partialsDir) => {
+const setupHandlebars = (...partialDirs) => {
   Handlebars.registerHelper('obj', ({ hash }) => hash)
   Handlebars.registerHelper('json', (c) => JSON.stringify(c))
   Handlebars.registerHelper('del', () => JSON.stringify(DELETE))
 
-  // Load all content files as camelcase partial names
-  for (const f of fs.readdirSync(join(partialsDir))) {
-    Handlebars.registerPartial(
-      partialName(f),
-      fs.readFileSync(join(partialsDir, f)).toString()
-    )
+  // Load all files as camelcase partial names.
+  // all other content dirs only get special underscore leading
+  // files as partials. this prevents recursion loops when overwriting
+  // a filename to use as a enw file
+  let isBase = true
+  for (const dir of partialDirs) {
+    for (const f of fs.readdirSync(dir)) {
+      if (f.startsWith('_') || isBase) {
+        Handlebars.registerPartial(
+          partialName(f),
+          fs.readFileSync(join(dir, f)).toString()
+        )
+      }
+    }
+    isBase = false
   }
 }
 
-const cache = new Map()
-
 const template = (str, { config, ...options }) => {
-  if (cache.size === 0) {
-    setupHandlebars(config.sourceDir)
-  }
+  setupHandlebars(...config.__PARTIAL_DIRS__)
 
-  let t = cache.get(str)
-  if (t == null) {
-    t = Handlebars.compile(str, { strict: true })
-    cache.set(str, t)
-  }
+  const t = Handlebars.compile(str, { strict: true })
 
   // merge in config as top level data in templates
   return t({ ...options, ...config })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@npmcli/template-oss",
-  "version": "4.1.2",
+  "version": "4.2.0",
   "description": "templated files used in npm CLI team oss projects",
   "main": "lib/content/index.js",
   "bin": {
@@ -42,6 +42,7 @@
     "handlebars": "^4.7.7",
     "hosted-git-info": "^5.0.0",
     "json-parse-even-better-errors": "^2.3.1",
+    "just-deep-map-values": "^1.1.1",
     "just-diff": "^5.0.1",
     "lodash": "^4.17.21",
     "npm-package-arg": "^9.0.1",
@@ -60,7 +61,11 @@
     "tap": "^16.0.0"
   },
   "tap": {
-    "timeout": 600
+    "timeout": 600,
+    "nyc-arg": [
+      "--exclude",
+      "tap-snapshots/**"
+    ]
   },
   "templateOSS": {
     "//@npmcli/template-oss": "This file is partially managed by @npmcli/template-oss. Edits may be overwritten."

package/lib/content/release-test.yml

@@ -1,46 +0,0 @@
-name: Release
-
-on:
-  workflow_call:
-    inputs:
-      ref:
-        required: true
-        type: string
-
-jobs:
-  lint-all:
-    runs-on: ubuntu-latest
-    steps:
-      {{> setupGit with=(obj ref="${{ inputs.ref }}")}}
-      {{> setupNode}}
-      {{> setupDeps}}
-      - run: npm run lint --if-present --workspaces --include-workspace-root
-
-  test-all:
-    strategy:
-      fail-fast: false
-      matrix:
-        node-version:
-          {{#each ciVersions}}
-          - {{.}}
-          {{/each}}
-        platform:
-          - os: ubuntu-latest
-            shell: bash
-          - os: macos-latest
-            shell: bash
-          {{#if windowsCI}}
-          - os: windows-latest
-            shell: cmd
-          {{/if}}
-    runs-on: $\{{ matrix.platform.os }}
-    defaults:
-      run:
-        shell: $\{{ matrix.platform.shell }}
-    steps:
-      {{> setupGit with=(obj ref="${{ inputs.ref }}")}}
-      {{> setupNode useMatrix=true}}
-      {{> setupDeps}}
-      - name: add tap problem matcher
-        run: echo "::add-matcher::.github/matchers/tap.json"
-      - run: npm run test --if-present --workspaces --include-workspace-root