@npmcli/template-oss 4.19.0 → 4.20.0

package/README.md

@@ -3,7 +3,11 @@
 This module bundles the npm CLI team's basics for package development into a
 single devDependency.
 
-**CAUTION: THESE CHANGES WILL OVERWRITE ANY LOCAL FILES AND SETTINGS**
+> [!WARNING]  
+> THESE CHANGES WILL OVERWRITE LOCAL FILES AND SETTINGS
+
+> [!IMPORTANT]  
+> This package does not follow semantic versioning for its API. This package is designed to be installed with `--save-exact` and therefore will make breaking API changes outside of major versions, including `engines` narrowing. Major versions are reserved for breaking changes to files written to a repo by this package.
 
 ### Configuration
 
@@ -60,21 +64,21 @@ any of the same settings in the root.
 ### Content
 
 All the templated content for this repo lives in
-[`lib/content/`](./lib/content/). The `index.js`[./lib/content/index.js] file
+[`lib/content/`](./lib/content/). The [`index.js`](./lib/content/index.js) file
 controls how and where this content is written.
 
 Content files can be overwritten or merged with the existing target file.
-Currently mergining is only supported for `package.json` files.
+Merging is only supported for some types of files (ini, yaml, json, package.json)
 
 Each content file goes through the following pipeline:
 
 1. It is read from its source location
-1. It is are templated using Handlebars with the variables from each packages's
-   config (with some derived values generated in [`config.js`](./lib/config.js)
-1. It is parsed based on its file extension in
+2. It is compiled using [Handlebars](https://handlebarsjs.com/) with the variables from each packages's
+   config (with some derived values generated in [`config.js`](./lib/config.js))
+3. It is parsed based on its file extension in
    [`parser.js`](./lib/util/parser.js)
-1. Additional logic is applied by the parser
-1. It is written to its target location
+4. Additional logic is applied by the parser
+5. It is written to its target location
 
 ### Usage
 
@@ -82,7 +86,7 @@ This package provides two bin scripts:
 
 #### `template-oss-check`
 
-This will check if any of the applied files different from the target content,
+This will check if any of the applied files differ from the target content,
 or if any of the other associated checks fail. The diffs of each file or check
 will be reported with instructions on how to fix it.
 
@@ -92,7 +96,7 @@ This will write all source files to their target locations in the cwd. It will
 do nothing if `package.json#templateOSS.version` is the same as the version
 being run. This can be overridden by `--force`.
 
-This is the script that is run on `postinsall`.
+This is the script that is run on `postinstall`.
 
 ### Extending
 
@@ -100,7 +104,7 @@ This is the script that is run on `postinsall`.
 
 This directory is where all the logic for applying files lives. It should be
 possible to add new files without modifying anything in this directory. To add a
-file, add the templated file to `lib/content/$FILENAME` and add entry for it in
+file, add the templated file to `lib/content/$FILENAME` and an entry for it in
 `lib/content/index.js` depending on where and when it should be written (root vs
 workspace, repo vs module, add vs remove, etc).
 

package/bin/release-manager.js

@@ -50,6 +50,12 @@ const MANUAL_PUBLISH_STEPS = `
 `
 
 const AUTO_PUBLISH_STEPS = `
+1. Approve this PR
+
+    \`\`\`sh
+    gh pr review <PR-NUMBER> -R {NWO} --approve
+    \`\`\`
+
 1. Merge release PR :rotating_light: Merging this will auto publish :rotating_light:
 
     \`\`\`sh

package/lib/config.js

@@ -1,18 +1,22 @@
-const { relative, dirname, join, extname, posix, win32 } = require('path')
-const { defaults, pick, omit, uniq, isPlainObject } = require('lodash')
+const { relative, dirname, join, extname } = require('path')
+const { defaults, defaultsDeep, pick, omit, uniq, isPlainObject } = require('lodash')
+const semver = require('semver')
 const ciVersions = require('./util/ci-versions.js')
 const parseDependabot = require('./util/dependabot.js')
 const git = require('./util/git.js')
 const gitignore = require('./util/gitignore.js')
 const { mergeWithCustomizers, customizers } = require('./util/merge.js')
 const { FILE_KEYS, parseConfig: parseFiles, getAddedFiles, mergeFiles } = require('./util/files.js')
+const template = require('./util/template.js')
+const getCmdPath = require('./util/get-cmd-path.js')
+const importOrRequire = require('./util/import-or-require.js')
+const { makePosix, deglob, posixDir, posixGlob } = require('./util/path.js')
+const { name: NAME, version: LATEST_VERSION } = require('../package.json')
 
 const CONFIG_KEY = 'templateOSS'
-const getPkgConfig = (pkg) => pkg[CONFIG_KEY] || {}
-
-const { name: NAME, version: LATEST_VERSION } = require('../package.json')
 const MERGE_KEYS = [...FILE_KEYS, 'defaultContent', 'content']
 const DEFAULT_CONTENT = require.resolve(NAME)
+const getPkgConfig = (pkg) => pkg[CONFIG_KEY] || {}
 
 const merge = mergeWithCustomizers(
   customizers.mergeArrays('branches', 'distPaths', 'allowPaths', 'ignorePaths'),
@@ -23,43 +27,6 @@ const merge = mergeWithCustomizers(
   }
 )
 
-const makePosix = (v) => v.split(win32.sep).join(posix.sep)
-const deglob = (v) => makePosix(v).replace(/[/*]+$/, '')
-const posixDir = (v) => `${v === '.' ? '' : deglob(v).replace(/\/$/, '')}${posix.sep}`
-const posixGlob = (str) => `${posixDir(str)}**`
-
-const getCmdPath = (key, { pkgConfig, rootConfig, isRoot, pkg, rootPkg }) => {
-  const result = (local, isRelative) => {
-    let root = local
-    const isLocal = local.startsWith('.') || local.startsWith('/')
-
-    if (isLocal) {
-      if (isRelative) {
-        // Make a path relative from a workspace to the root if we are in a workspace
-        local = makePosix(join(relative(pkg.path, rootPkg.path), local))
-      }
-      local = `node ${local}`
-      root = `node ${root}`
-    }
-
-    return {
-      isLocal,
-      local,
-      root,
-    }
-  }
-
-  if (pkgConfig[key]) {
-    return result(pkgConfig[key])
-  }
-
-  if (rootConfig[key]) {
-    return result(rootConfig[key], !isRoot)
-  }
-
-  return result(key)
-}
-
 const mergeConfigs = (...configs) => {
   const mergedConfig = merge(...configs.map(c => pick(c, MERGE_KEYS)))
   return defaults(mergedConfig, {
@@ -72,37 +39,33 @@ const mergeConfigs = (...configs) => {
   })
 }
 
-const readContentPath = (path) => {
+const readContentPath = async (path) => {
   if (!path) {
     return {}
   }
 
-  let content = {}
   const index = extname(path) === '.js' ? path : join(path, 'index.js')
   const dir = dirname(index)
-
-  try {
-    content = require(index)
-  } catch {
-    // its ok if this fails since the content dir
-    // might only be to provide other files. the
-    // index.js is optional
-  }
+  const content = await importOrRequire(index)
 
   return { content, dir }
 }
 
-const getConfig = (path, rawConfig) => {
-  const config = omit(readContentPath(path).content, FILE_KEYS)
+const getConfig = async (path, rawConfig) => {
+  const { content } = await readContentPath(path)
+  const config = omit(content, FILE_KEYS)
   return merge(config, rawConfig ? omit(rawConfig, FILE_KEYS) : {})
 }
 
-const getFiles = (path, rawConfig) => {
-  const { content, dir } = readContentPath(path)
+const getFiles = async (path, rawConfig, templateSettings) => {
+  const { content, dir } = await readContentPath(path)
   if (!dir) {
     return []
   }
-  return [parseFiles(pick(content, FILE_KEYS), dir, pick(rawConfig, FILE_KEYS)), dir]
+  return [
+    parseFiles(pick(content, FILE_KEYS), dir, pick(rawConfig, FILE_KEYS), templateSettings),
+    dir,
+  ]
 }
 
 const getFullConfig = async ({
@@ -127,39 +90,15 @@ const getFullConfig = async ({
   // These config items are merged betweent the root and child workspaces and only come from
   // the package.json because they can be used to read configs from other the content directories
   const mergedConfig = mergeConfigs(rootPkg.config, pkg.config)
-
-  const defaultConfig = getConfig(DEFAULT_CONTENT)
-  const [defaultFiles, defaultDir] = getFiles(DEFAULT_CONTENT, mergedConfig)
+  const defaultConfig = await getConfig(DEFAULT_CONTENT)
   const useDefault = mergedConfig.defaultContent && defaultConfig
 
-  const rootConfig = getConfig(rootPkg.config.content, rootPkg.config)
-  const [rootFiles, rootDir] = getFiles(rootPkg.config.content, mergedConfig)
+  const rootConfig = await getConfig(rootPkg.config.content, rootPkg.config)
 
   // The content config only gets set from the package we are in, it doesn't inherit
   // anything from the root
   const rootPkgConfig = merge(useDefault, rootConfig)
-  const pkgConfig = merge(useDefault, getConfig(pkg.config.content, pkg.config))
-  const [pkgFiles, pkgDir] = getFiles(mergedConfig.content, mergedConfig)
-
-  // Files get merged in from the default content (that template-oss provides) as well
-  // as any content paths provided from the root or the workspace
-  const fileDirs = uniq([useDefault && defaultDir, rootDir, pkgDir].filter(Boolean))
-  const files = mergeFiles(useDefault && defaultFiles, rootFiles, pkgFiles)
-  const repoFiles = isRoot ? files.rootRepo : files.workspaceRepo
-  const moduleFiles = isRoot ? files.rootModule : files.workspaceModule
-
-  const allowRootDirs = [
-    // Allways allow module files in root or workspaces
-    ...getAddedFiles(moduleFiles),
-    ...isRoot ? [
-      // in the root allow all repo files
-      ...getAddedFiles(repoFiles),
-      // and allow all workspace repo level files in the root
-      ...pkgs
-        .filter(p => p.path !== rootPkg.path && p.config.workspaceRepo !== false)
-        .flatMap(() => getAddedFiles(files.workspaceRepo)),
-    ] : [],
-  ]
+  const pkgConfig = merge(useDefault, await getConfig(pkg.config.content, pkg.config))
 
   const npmPath = getCmdPath('npm', { pkgConfig, rootConfig, isRoot, pkg, rootPkg })
   const npxPath = getCmdPath('npx', { pkgConfig, rootConfig, isRoot, pkg, rootPkg })
@@ -185,6 +124,8 @@ const getFullConfig = async ({
     ? pkgConfig.releaseBranch.replace(/\*/g, pkgConfig.backport)
     : defaultBranch
 
+  const esm = pkg.pkgJson?.type === 'module' || !!pkgConfig.typescript || !!pkgConfig.esm
+
   // all derived keys
   const derived = {
     isRoot,
@@ -209,18 +150,11 @@ const getFullConfig = async ({
     releaseBranch,
     publishTag,
     dependabot: parseDependabot(pkgConfig, defaultConfig, gitBranches.branches),
-    // repo
+    // paths
     repoDir: rootPkg.path,
-    repoFiles,
-    applyRepo: !!repoFiles,
-    // module
     moduleDir: pkg.path,
-    moduleFiles,
-    applyModule: !!moduleFiles,
-    // package
     pkgName: pkg.pkgJson.name,
     pkgNameFs: pkg.pkgJson.name.replace(/\//g, '-').replace(/@/g, ''),
-    // paths
     pkgPath,
     pkgDir: posixDir(pkgPath),
     pkgGlob: posixGlob(pkgPath),
@@ -228,6 +162,12 @@ const getFullConfig = async ({
     allFlags: isMono ? '-ws -iwr --if-present' : '',
     workspacePaths,
     workspaceGlobs: workspacePaths.map(posixGlob),
+    // type
+    esm,
+    cjsExt: esm ? 'cjs' : 'js',
+    deleteJsExt: esm ? 'js' : 'cjs',
+    // tap
+    tap18: semver.coerce(pkg.pkgJson?.devDependencies?.tap)?.major === 18,
     // booleans to control application of updates
     isForce,
     isDogFood,
@@ -243,20 +183,6 @@ const getFullConfig = async ({
     lockfile: rootPkgConfig.lockfile,
     // ci versions / engines
     ciVersions: ciVersions.get(pkg.pkgJson.engines?.node, pkgConfig),
-    // gitignore
-    ignorePaths: [
-      ...gitignore.sort([
-        ...gitignore.allowRootDir(allowRootDirs),
-        ...isRoot && pkgConfig.lockfile ? ['!/package-lock.json'] : [],
-        ...(pkgConfig.allowPaths || []).map((p) => `!${p}`),
-        ...(pkgConfig.ignorePaths || []),
-      ]),
-      // these cant be sorted since they rely on order
-      // to allow a previously ignored directoy
-      ...isRoot
-        ? gitignore.allowDir(wsPkgs.map((p) => makePosix(relative(rootPkg.path, p.path))))
-        : [],
-    ],
     // needs update if we are dogfooding this repo, with force argv, or its
     // behind the current version
     needsUpdate: isForce || isDogFood || !isLatest,
@@ -264,15 +190,22 @@ const getFullConfig = async ({
     __NAME__: NAME,
     __CONFIG_KEY__: CONFIG_KEY,
     __VERSION__: LATEST_VERSION,
-    __PARTIAL_DIRS__: fileDirs,
   }
 
-  if (!pkgConfig.eslint) {
-    derived.ignorePaths = derived.ignorePaths.filter(p => !p.includes('eslint'))
-    if (Array.isArray(pkgConfig.requiredPackages?.devDependencies)) {
-      pkgConfig.requiredPackages.devDependencies =
-        pkgConfig.requiredPackages.devDependencies.filter(p => !p.includes('eslint'))
-    }
+  if (!pkgConfig.eslint && Array.isArray(pkgConfig.requiredPackages?.devDependencies)) {
+    pkgConfig.requiredPackages.devDependencies =
+      pkgConfig.requiredPackages.devDependencies.filter(p => !p.includes('eslint'))
+  }
+
+  if (pkgConfig.typescript) {
+    defaultsDeep(pkgConfig, { allowPaths: [], requiredPackages: { devDependencies: [] } })
+    pkgConfig.distPaths = null
+    pkgConfig.allowPaths.push('/src/')
+    pkgConfig.requiredPackages.devDependencies.push(
+      'typescript',
+      'tshy',
+      '@typescript-eslint/parser'
+    )
   }
 
   const gitUrl = await git.getUrl(rootPkg.path)
@@ -284,10 +217,55 @@ const getFullConfig = async ({
     }
   }
 
-  return {
-    ...pkgConfig,
-    ...derived,
-  }
+  const fullConfig = { ...pkgConfig, ...derived }
+
+  // files, come at the end since file names can be based on config
+  const [defaultFiles, defaultDir] = await getFiles(DEFAULT_CONTENT, mergedConfig, fullConfig)
+  const [rootFiles, rootDir] = await getFiles(rootPkg.config.content, mergedConfig, fullConfig)
+  const [pkgFiles, pkgDir] = await getFiles(mergedConfig.content, mergedConfig, fullConfig)
+
+  // Files get merged in from the default content (that template-oss provides) as well
+  // as any content paths provided from the root or the workspace
+  const fileDirs = uniq([useDefault && defaultDir, rootDir, pkgDir].filter(Boolean))
+  const files = mergeFiles(useDefault && defaultFiles, rootFiles, pkgFiles)
+  const repoFiles = isRoot ? files.rootRepo : files.workspaceRepo
+  const moduleFiles = isRoot ? files.rootModule : files.workspaceModule
+
+  Object.assign(fullConfig, {
+    repoFiles,
+    moduleFiles,
+    applyRepo: !!repoFiles,
+    applyModule: !!moduleFiles,
+    __PARTIAL_DIRS__: fileDirs,
+    // gitignore, these use the full config so need to come at the very end
+    ignorePaths: [
+      ...gitignore.sort([
+        ...gitignore.allowRootDir([
+          // Allways allow module files in root or workspaces
+          ...getAddedFiles(moduleFiles).map(s => template(s, fullConfig)),
+          ...isRoot ? [
+            // in the root allow all repo files
+            ...getAddedFiles(repoFiles).map(s => template(s, fullConfig)),
+            // and allow all workspace repo level files in the root
+            ...pkgs
+              .filter(p => p.path !== rootPkg.path && p.config.workspaceRepo !== false)
+              .flatMap(() => getAddedFiles(files.workspaceRepo)),
+          ] : [],
+        ]),
+        ...isRoot && pkgConfig.lockfile ? ['!/package-lock.json'] : [],
+        ...(pkgConfig.allowPaths || []).map((p) => `!${p}`),
+        ...(pkgConfig.distPaths || []).map((p) => `!/${p}`),
+        ...(pkgConfig.ignorePaths || []),
+      ]),
+      // these cant be sorted since they rely on order
+      // to allow a previously ignored directoy
+      ...isRoot
+        ? gitignore.allowDir(wsPkgs.map((p) => makePosix(relative(rootPkg.path, p.path))))
+        : [],
+    ].filter(p => !pkgConfig.eslint ? !p.includes('eslint') : true),
+  })
+
+  return fullConfig
 }
 
 module.exports = getFullConfig

package/lib/content/{_job-matrix.yml → _job-matrix-yml.hbs}

@@ -26,4 +26,4 @@ defaults:
   run:
     shell: $\{{ matrix.platform.shell }}
 steps:
-  {{> stepsSetup jobIsMatrix=true }}
+  {{> stepsSetupYml jobIsMatrix=true }}

package/lib/content/{_job-release-integration.yml → _job-release-integration-yml.hbs}

@@ -29,7 +29,7 @@ defaults:
   run:
     shell: bash
 steps:
-  {{> stepNode lockfile=false }}
+  {{> stepNodeYml lockfile=false }}
   - name: View in Registry
     run: |
       EXIT_CODE=0

package/lib/content/{_job.yml → _job-yml.hbs}

@@ -5,4 +5,4 @@ defaults:
   run:
     shell: bash
 steps:
-  {{> stepsSetup }}
+  {{> stepsSetupYml }}

package/lib/content/_step-node-yml.hbs

@@ -0,0 +1,60 @@
+- name: Setup Node
+  uses: actions/setup-node@v3
+  id: node
+  with:
+    node-version: {{#if jobIsMatrix}}$\{{ matrix.node-version }}{{else}}{{ last ciVersions }}{{/if}}
+    check-latest: contains({{#if jobIsMatrix}}matrix.node-version{{else}}'{{ last ciVersions }}'{{/if}}, '.x')
+    {{#if lockfile}}
+    cache: npm
+    {{/if}}
+
+{{#if updateNpm}}
+{{#if jobIsMatrix}}
+# node 10/12/14 ship with npm@6, which is known to fail when updating itself in windows
+- name: Update Windows npm
+  if: |
+    matrix.platform.os == 'windows-latest' && (
+      startsWith(steps.node.outputs.node-version, 'v10.') ||
+      startsWith(steps.node.outputs.node-version, 'v12.') ||
+      startsWith(steps.node.outputs.node-version, 'v14.')
+    )
+  run: |
+    curl -sO https://registry.npmjs.org/npm/-/npm-7.5.4.tgz
+    tar xf npm-7.5.4.tgz
+    cd package
+    node lib/npm.js install --no-fund --no-audit -g ..\npm-7.5.4.tgz
+    cd ..
+    rmdir /s /q package
+{{/if}}
+
+- name: Install Latest npm
+  shell: bash
+  env:
+    NODE_VERSION: $\{{ steps.node.outputs.node-version }}
+  run: |
+    MATCH=""
+    SPECS=("latest" "next-10" "next-9" "next-8" "next-7" "next-6")
+
+    echo "node@$NODE_VERSION"
+
+    for SPEC in ${SPECS[@]}; do
+      ENGINES=$(npm view npm@$SPEC --json | jq -r '.engines.node')
+      echo "Checking if node@$NODE_VERSION satisfies npm@$SPEC ($ENGINES)"
+
+      if npx semver -r "$ENGINES" "$NODE_VERSION" > /dev/null; then
+        MATCH=$SPEC
+        echo "Found compatible version: npm@$MATCH"
+        break
+      fi  
+    done
+
+    if [ -z $MATCH ]; then
+      echo "Could not find a compatible version of npm for node@$NODE_VERSION"
+      exit 1
+    fi
+
+    npm i --prefer-online --no-fund --no-audit -g npm@$MATCH
+
+- name: npm Version
+  run: npm -v
+{{/if}}

package/lib/content/_steps-setup-yml.hbs

@@ -0,0 +1,6 @@
+{{~#if jobCheck}}{{> stepChecksYml }}{{/if}}
+{{~#unless jobSkipSetup}}
+{{> stepGitYml }}
+{{> stepNodeYml }}
+{{> stepDepsYml }}
+{{/unless}}

package/lib/content/{audit.yml → audit-yml.hbs}

@@ -8,5 +8,5 @@ on:
 
 jobs:
   audit:
-    {{> job jobName="Audit Dependencies" jobDepFlags="--package-lock" }}
-      {{> stepAudit }}
+    {{> jobYml jobName="Audit Dependencies" jobDepFlags="--package-lock" }}
+      {{> stepAuditYml }}

package/lib/content/{ci-release.yml → ci-release-yml.hbs}

@@ -19,19 +19,19 @@ on:
 
 jobs:
   lint-all:
-    {{> job
+    {{> jobYml
       jobName="Lint All"
       jobCheck=(obj sha="inputs.check-sha")
       jobCheckout=(obj ref="${{ inputs.ref }}")
     }}
-      {{> stepLint jobRunFlags=allFlags }}
-      {{> stepChecks jobCheck=true }}
+      {{> stepLintYml jobRunFlags=allFlags }}
+      {{> stepChecksYml jobCheck=true }}
 
   test-all:
-    {{> jobMatrix
+    {{> jobMatrixYml
       jobName="Test All"
       jobCheck=(obj sha="inputs.check-sha")
       jobCheckout=(obj ref="${{ inputs.ref }}")
     }}
-      {{> stepTest jobRunFlags=allFlags }}
-      {{> stepChecks jobCheck=true }}
+      {{> stepTestYml jobRunFlags=allFlags }}
+      {{> stepChecksYml jobCheck=true }}

package/lib/content/ci-yml.hbs

@@ -0,0 +1,13 @@
+name: CI {{~#if isWorkspace}} - {{ pkgName }}{{/if}}
+
+on:
+  {{> onCiYml }}
+
+jobs:
+  lint:
+    {{> jobYml jobName="Lint" }}
+      {{> stepLintYml jobRunFlags=pkgFlags }}
+
+  test:
+    {{> jobMatrixYml jobName="Test" }}
+      {{> stepTestYml jobRunFlags=pkgFlags }}

package/lib/content/{codeql-analysis.yml → codeql-analysis-yml.hbs}

@@ -24,7 +24,7 @@ jobs:
       contents: read
       security-events: write
     steps:
-      {{> stepGit }}
+      {{> stepGitYml }}
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2
         with:

package/lib/content/{eslintrc.js → eslintrc-js.hbs}

@@ -8,12 +8,22 @@ const localConfigs = readdir(__dirname)
 
 module.exports = {
   root: true,
-  {{#if workspaceGlobs}}
   ignorePatterns: [
+    'tap-testdir*/',
     {{#each workspaceGlobs}}
     '{{ . }}',
     {{/each}}
+    {{#if typescript}}
+    'dist/',
+    {{/if}}
   ],
+  {{#if typescript}}
+  parser: '@typescript-eslint/parser',
+  settings: {
+    'import/resolver': {
+      typescript: {},
+    },
+  },
   {{/if}}
   extends: [
     '@npmcli',

package/lib/content/{gitignore → gitignore.hbs}

@@ -1,5 +1,7 @@
 # ignore everything in the root
 /*
+# transient test directories
+tap-testdir*/
 
 # keep these
 {{#each ignorePaths}}