@antora/content-aggregator 3.0.0-alpha.9 → 3.0.0-beta.4

package/README.md

@@ -3,7 +3,7 @@
 The Content Aggregator is a component in Antora responsible for fetching and aggregating content distributed across multiple local and remote git repositories for use in an Antora documentation pipeline.
 
 [Antora](https://antora.org) is a modular static site generator designed for creating documentation sites from AsciiDoc documents.
-Its site generator pipeline aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
+Its site generator aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
 
 ## Copyright and License
 

package/lib/aggregate-content.js

@@ -1,7 +1,5 @@
 'use strict'
 
-if (!Promise.allSettled) require('./promise-all-settled-polyfill')
-
 const camelCaseKeys = require('camelcase-keys')
 const { createHash } = require('crypto')
 const createHttpPlugin = require('./git-plugin-http')
@@ -9,6 +7,7 @@ const decodeUint8Array = require('./decode-uint8-array')
 const EventEmitter = require('events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
+const filterRefs = require('./filter-refs')
 const flattenDeep = require('./flatten-deep')
 const fs = require('fs')
 const { promises: fsp } = fs
@@ -16,18 +15,17 @@ const getCacheDir = require('cache-directory')
 const GitCredentialManagerStore = require('./git-credential-manager-store')
 const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
-const invariably = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }
-const { makeRe: makePicomatchRx } = require('picomatch')
-const matcher = require('matcher')
-const MultiProgress = require('multi-progress')
+const globStream = require('glob-stream')
+const invariably = require('./invariably')
+const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')
+const MultiProgress = require('multi-progress') // calls require('progress') as a peer dependencies
 const ospath = require('path')
 const { posix: path } = ospath
 const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
 const { fs: resolvePathGlobsFs, git: resolvePathGlobsGit } = require('./resolve-path-globs')
-const { Transform } = require('stream')
-const map = (transform, flush = undefined) => new Transform({ objectMode: true, transform, flush })
+const { pipeline, Writable } = require('stream')
+const forEach = (write) => new Writable({ objectMode: true, write })
 const userRequire = require('@antora/user-require-helper')
-const vfs = require('vinyl-fs')
 const yaml = require('js-yaml')
 
 const {
@@ -39,11 +37,10 @@ const {
   GIT_CORE,
   GIT_OPERATION_LABEL_LENGTH,
   GIT_PROGRESS_PHASES,
-  PICOMATCH_VERSION_OPTS,
+  REF_PATTERN_CACHE_KEY,
   SYMLINK_FILE_MODE,
   VALID_STATE_FILENAME,
 } = require('./constants')
-
 const ANY_SEPARATOR_RX = /[:/]/
 const CSV_RX = /\s*,\s*/
 const VENTILATED_CSV_RX = /\s*,\s+/
@@ -101,7 +98,8 @@ function aggregateContent (playbook) {
       return accum.set(source.url, [...(accum.get(source.url) || []), Object.assign({}, sourceDefaults, source)])
     }, new Map())
     const progress = !quiet && createProgress(sourcesByUrl.keys(), process.stdout)
-    const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir }
+    const refPatternCache = Object.assign(new Map(), { braces: new Map() })
+    const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir, refPatternCache }
     return collectFiles(sourcesByUrl, loadOpts, fetchConcurrency).then(buildAggregate, (err) => {
       progress && progress.terminate()
       throw err
@@ -122,14 +120,14 @@ async function collectFiles (sourcesByUrl, loadOpts, concurrency) {
         })
       ),
   ])
-  let rejected, started
+  let rejection, started
   const startedContinuations = []
   const recordRejection = (err) => {
-    throw (rejected = true) && err
+    rejection = err
   }
   const runTask = (primary, continuation, idx) =>
     primary().then((value) => {
-      if (!rejected) startedContinuations[idx] = continuation(value).catch(recordRejection)
+      if (!rejection) startedContinuations[idx] = continuation(value).catch(recordRejection)
     }, recordRejection)
   if (tasks.length > concurrency) {
     started = []
@@ -140,17 +138,16 @@ async function collectFiles (sourcesByUrl, loadOpts, concurrency) {
       )
       started.push(current)
       if (pending.push(current) < concurrency) continue
-      if (await Promise.race(pending).then(invariably.true, invariably.false)) continue
-      break
+      await Promise.race(pending)
+      if (rejection) break
     }
   } else {
     started = tasks.map(([primary, continuation], idx) => runTask(primary, continuation, idx))
   }
-  return Promise.allSettled(started).then((outcomes) =>
-    Promise.allSettled(startedContinuations).then((continuationOutcomes) => {
-      const rejection = outcomes.push(...continuationOutcomes) && outcomes.find(({ status }) => status === 'rejected')
-      if (rejection) throw rejection.reason
-      return continuationOutcomes.map(({ value }) => value)
+  return Promise.all(started).then(() =>
+    Promise.all(startedContinuations).then((result) => {
+      if (rejection) throw rejection
+      return result
     })
   )
 }
@@ -173,13 +170,14 @@ function buildAggregate (componentVersionBuckets) {
 
 async function loadRepository (url, opts) {
   let authStatus, dir, repo
+  const cache = { [REF_PATTERN_CACHE_KEY]: opts.refPatternCache }
   if (~url.indexOf(':') && GIT_URI_DETECTOR_RX.test(url)) {
     let credentials, displayUrl
     ;({ displayUrl, url, credentials } = extractCredentials(url))
     const { cacheDir, fetch, fetchTags, gitPlugins, progress } = opts
     dir = ospath.join(cacheDir, generateCloneFolderName(displayUrl))
     // NOTE the presence of the url property on the repo object implies the repository is remote
-    repo = { cache: {}, dir, fs, gitdir: dir, noCheckout: true, url }
+    repo = { cache, dir, fs, gitdir: dir, noCheckout: true, url }
     const validStateFile = ospath.join(dir, VALID_STATE_FILENAME)
     try {
       await fsp.access(validStateFile)
@@ -204,7 +202,7 @@ async function loadRepository (url, opts) {
         authStatus = await git.getConfig(Object.assign({ path: 'remote.origin.private' }, repo))
       }
     } catch (gitErr) {
-      await rmdir(dir)
+      await fsp['rm' in fsp ? 'rm' : 'rmdir'](dir, { recursive: true, force: true })
       if (gitErr.rethrow) throw transformGitCloneError(gitErr, displayUrl)
       const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetchTags, 'clone')
       await git
@@ -225,15 +223,12 @@ async function loadRepository (url, opts) {
     }
   } else if (await isDirectory((dir = expandPath(url, { dot: opts.startDir })))) {
     const gitdir = ospath.join(dir, '.git')
-    repo = (await isDirectory(gitdir))
-      ? { cache: {}, dir, fs, gitdir }
-      : { cache: {}, dir, fs, gitdir: dir, noCheckout: true }
+    repo = (await isDirectory(gitdir)) ? { cache, dir, fs, gitdir } : { cache, dir, fs, gitdir: dir, noCheckout: true }
     try {
       await git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo))
     } catch {
-      throw new Error(
-        `Local content source must be a git repository: ${dir}${url !== dir ? ' (url: ' + url + ')' : ''}`
-      )
+      const msg = `Local content source must be a git repository: ${dir}${url !== dir ? ' (url: ' + url + ')' : ''}`
+      throw new Error(msg)
     }
   } else {
     throw new Error(`Local content source does not exist: ${dir}${url !== dir ? ' (url: ' + url + ')' : ''}`)
@@ -271,99 +266,102 @@ async function collectFilesFromSource (source, repo, remoteName, authStatus) {
 async function selectReferences (source, repo, remote) {
   let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns = '.' } = source
   const isBare = repo.noCheckout
+  const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   const noWorktree = repo.url ? undefined : null
   const refs = new Map()
-  if (tagPatterns) {
-    tagPatterns = Array.isArray(tagPatterns)
+  if (
+    tagPatterns &&
+    (tagPatterns = Array.isArray(tagPatterns)
       ? tagPatterns.map((pattern) => String(pattern))
-      : String(tagPatterns).split(CSV_RX)
-    if (tagPatterns.length) {
-      const tags = await git.listTags(repo)
-      for (const shortname of tags.length ? matcher(tags, tagPatterns) : tags) {
+      : splitRefPatterns(String(tagPatterns))).length
+  ) {
+    const tags = await git.listTags(repo)
+    if (tags.length) {
+      for (const shortname of filterRefs(tags, tagPatterns, patternCache)) {
         // NOTE tags are stored using symbol keys to distinguish them from branches
         refs.set(Symbol(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
       }
     }
   }
-  if (branchPatterns) {
-    if (worktreePatterns) {
-      if (worktreePatterns === '.') {
-        worktreePatterns = ['.']
-      } else if (worktreePatterns === true) {
-        worktreePatterns = ['.', '*']
-      } else {
-        worktreePatterns = Array.isArray(worktreePatterns)
-          ? worktreePatterns.map((pattern) => String(pattern))
-          : String(worktreePatterns).split(CSV_RX)
-      }
+  if (!branchPatterns) return [...refs.values()]
+  if (worktreePatterns) {
+    if (worktreePatterns === '.') {
+      worktreePatterns = ['.']
+    } else if (worktreePatterns === true) {
+      worktreePatterns = ['.', '*']
+    } else {
+      worktreePatterns = Array.isArray(worktreePatterns)
+        ? worktreePatterns.map((pattern) => String(pattern))
+        : splitRefPatterns(String(worktreePatterns))
     }
-    const branchPatternsString = String(branchPatterns)
-    if (branchPatternsString === 'HEAD' || branchPatternsString === '.') {
+  }
+  const branchPatternsString = String(branchPatterns)
+  if (branchPatternsString === 'HEAD' || branchPatternsString === '.') {
+    const currentBranch = await getCurrentBranchName(repo, remote)
+    if (currentBranch) {
+      branchPatterns = [currentBranch]
+    } else if (isBare) {
+      return [...refs.values()]
+    } else {
+      // NOTE current branch is undefined when HEAD is detached
+      const head = worktreePatterns[0] === '.' ? repo.dir : noWorktree
+      refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
+      return [...refs.values()]
+    }
+  } else if (
+    (branchPatterns = Array.isArray(branchPatterns)
+      ? branchPatterns.map((pattern) => String(pattern))
+      : splitRefPatterns(branchPatternsString)).length
+  ) {
+    let headBranchIdx
+    // NOTE we can assume at least two entries if HEAD or . are present
+    if (~(headBranchIdx = branchPatterns.indexOf('HEAD')) || ~(headBranchIdx = branchPatterns.indexOf('.'))) {
       const currentBranch = await getCurrentBranchName(repo, remote)
       if (currentBranch) {
-        branchPatterns = [currentBranch]
-      } else {
-        if (!isBare) {
-          // NOTE current branch is undefined when HEAD is detached
-          const head = worktreePatterns[0] === '.' ? repo.dir : noWorktree
-          refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
-        }
-        return [...refs.values()]
-      }
-    } else if (
-      (branchPatterns = Array.isArray(branchPatterns)
-        ? branchPatterns.map((pattern) => String(pattern))
-        : branchPatternsString.split(CSV_RX)).length
-    ) {
-      let headBranchIdx
-      // NOTE we can assume at least two entries if HEAD or . are present
-      if (~(headBranchIdx = branchPatterns.indexOf('HEAD')) || ~(headBranchIdx = branchPatterns.indexOf('.'))) {
-        const currentBranch = await getCurrentBranchName(repo, remote)
-        if (currentBranch) {
-          // NOTE ignore if current branch is already in list
-          if (~branchPatterns.indexOf(currentBranch)) {
-            branchPatterns.splice(headBranchIdx, 1)
-          } else {
-            branchPatterns[headBranchIdx] = currentBranch
-          }
-        } else {
-          if (!isBare) {
-            let head = noWorktree
-            if (worktreePatterns[0] === '.') {
-              worktreePatterns = worktreePatterns.slice(1)
-              head = repo.dir
-            }
-            // NOTE current branch is undefined when HEAD is detached
-            refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
-          }
+        // NOTE ignore if current branch is already in list
+        if (~branchPatterns.indexOf(currentBranch)) {
           branchPatterns.splice(headBranchIdx, 1)
+        } else {
+          branchPatterns[headBranchIdx] = currentBranch
         }
+      } else if (isBare) {
+        branchPatterns.splice(headBranchIdx, 1)
+      } else {
+        let head = noWorktree
+        if (worktreePatterns[0] === '.') {
+          worktreePatterns = worktreePatterns.slice(1)
+          head = repo.dir
+        }
+        // NOTE current branch is undefined when HEAD is detached
+        refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
+        branchPatterns.splice(headBranchIdx, 1)
       }
-    } else {
-      return [...refs.values()]
     }
-    // NOTE isomorphic-git includes HEAD in list of remote branches (see https://isomorphic-git.org/docs/listBranches)
-    const remoteBranches = (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
-    if (remoteBranches.length) {
-      for (const shortname of matcher(remoteBranches, branchPatterns)) {
-        const fullname = 'remotes/' + remote + '/' + shortname
-        refs.set(shortname, { shortname, fullname, type: 'branch', remote, head: noWorktree })
-      }
+  } else {
+    return [...refs.values()]
+  }
+  // NOTE isomorphic-git includes HEAD in list of remote branches (see https://isomorphic-git.org/docs/listBranches)
+  const remoteBranches = (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
+  if (remoteBranches.length) {
+    for (const shortname of filterRefs(remoteBranches, branchPatterns, patternCache)) {
+      const fullname = 'remotes/' + remote + '/' + shortname
+      refs.set(shortname, { shortname, fullname, type: 'branch', remote, head: noWorktree })
     }
-    // NOTE only consider local branches if repo has a worktree or there are no remote tracking branches
-    if (!isBare) {
-      const localBranches = await git.listBranches(repo)
-      if (localBranches.length) {
-        const worktrees = await findWorktrees(repo, worktreePatterns)
-        for (const shortname of matcher(localBranches, branchPatterns)) {
-          const head = worktrees.get(shortname) || noWorktree
-          refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
-        }
+  }
+  // NOTE only consider local branches if repo has a worktree or there are no remote tracking branches
+  if (!isBare) {
+    const localBranches = await git.listBranches(repo)
+    if (localBranches.length) {
+      const worktrees = await findWorktrees(repo, worktreePatterns)
+      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
+        const head = worktrees.get(shortname) || noWorktree
+        refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
       }
-    } else if (!remoteBranches.length) {
-      // QUESTION should local branches be used if the only remote branch is HEAD?
-      const localBranches = await git.listBranches(repo)
-      for (const shortname of localBranches.length ? matcher(localBranches, branchPatterns) : localBranches) {
+    }
+  } else if (!remoteBranches.length) {
+    const localBranches = await git.listBranches(repo)
+    if (localBranches.length) {
+      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
         refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head: noWorktree })
       }
     }
@@ -437,28 +435,11 @@ function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePa
 }
 
 function readFilesFromWorktree (worktreePath, startPath) {
-  const cwd = ospath.join(worktreePath, startPath)
+  const cwd = ospath.join(worktreePath, startPath, '.') // . shaves off trailing slash
   return fsp.stat(cwd).then(
-    (stat) => {
-      if (!stat.isDirectory()) throw new Error(`the start path '${startPath}' is not a directory`)
-      return new Promise((resolve, reject) =>
-        vfs
-          .src(CONTENT_SRC_GLOB, Object.assign({ cwd }, CONTENT_SRC_OPTS))
-          .on('error', (err) => {
-            if (err.code === 'ENOENT' && err.syscall === 'stat') {
-              try {
-                if (fs.lstatSync(err.path).isSymbolicLink()) {
-                  err.message = `Broken symbolic link detected at ${ospath.relative(cwd, err.path)}`
-                }
-              } catch {}
-            } else if (err.code === 'ELOOP') {
-              err.message = `Symbolic link cycle detected at ${ospath.relative(cwd, err.path)}`
-            }
-            reject(err)
-          })
-          .pipe(relativizeFiles())
-          .pipe(collectDataFromStream(resolve))
-      )
+    (startPathStat) => {
+      if (!startPathStat.isDirectory()) throw new Error(`the start path '${startPath}' is not a directory`)
+      return srcFs(cwd)
     },
     () => {
       throw new Error(`the start path '${startPath}' does not exist`)
@@ -466,40 +447,43 @@ function readFilesFromWorktree (worktreePath, startPath) {
   )
 }
 
-/**
- * Transforms the path of every file in the stream to a relative posix path.
- *
- * Applies a mapping function to all files in the stream so they end up with a
- * posixified path relative to the file's base instead of the filesystem root.
- * This mapper also filters out any directories (indicated by file.isNull())
- * that got caught up in the glob.
- */
-function relativizeFiles () {
-  return map((file, enc, next) => {
-    if (file.isNull()) {
-      next()
-    } else {
-      next(
-        null,
-        new File({
-          path: posixify ? posixify(file.relative) : file.relative,
-          contents: file.contents,
-          stat: file.stat,
-          src: { abspath: file.path },
-        })
-      )
-    }
-  })
-}
-
-function collectDataFromStream (done) {
-  const accum = []
-  return map(
-    (obj, _, next) => {
-      accum.push(obj)
-      next()
-    },
-    () => done(accum)
+function srcFs (cwd) {
+  const relpathStart = cwd.length + 1
+  return new Promise((resolve, reject, cache = Object.create(null), files = []) =>
+    pipeline(
+      globStream(CONTENT_SRC_GLOB, Object.assign({ cache, cwd }, CONTENT_SRC_OPTS)),
+      forEach(({ path: abspathPosix }, _, done) => {
+        if (Array.isArray(cache[abspathPosix])) return done() // detects some directories, but not all
+        const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
+        const relpath = abspath.substr(relpathStart)
+        symlinkAwareStat(abspath).then(
+          (stat) => {
+            if (stat.isDirectory()) return done() // detects remaining directories
+            fsp.readFile(abspath).then(
+              (contents) => {
+                files.push(new File({ path: posixify ? posixify(relpath) : relpath, contents, stat, src: { abspath } }))
+                done()
+              },
+              (readErr) => {
+                done(Object.assign(readErr, { message: readErr.message.replace(`'${abspath}'`, relpath) }))
+              }
+            )
+          },
+          (statErr) => {
+            if (statErr.symlink) {
+              statErr.message =
+                statErr.code === 'ELOOP'
+                  ? `Symbolic link cycle detected at ${relpath}`
+                  : `Broken symbolic link detected at ${relpath}`
+            } else {
+              statErr.message = statErr.message.replace(`'${abspath}'`, relpath)
+            }
+            done(statErr)
+          }
+        )
+      }),
+      (err) => (err ? reject(err) : resolve(files))
+    )
   )
 }
 
@@ -601,7 +585,7 @@ function readGitSymlink (repo, root, parent, { oid }, following) {
       let targetParent
       if (parent.dirname) {
         const dirname = parent.dirname + '/'
-        target = path.join(dirname, target)
+        target = path.join(dirname, target) // join doesn't remove trailing separator
         if (target.startsWith(dirname)) {
           target = target.substr(dirname.length)
           targetParent = parent
@@ -609,10 +593,12 @@ function readGitSymlink (repo, root, parent, { oid }, following) {
           targetParent = root
         }
       } else {
-        target = path.normalize(target)
+        target = path.normalize(target) // normalize doesn't remove trailing separator
         targetParent = root
       }
-      return readGitObjectAtPath(repo, root, targetParent, target.split('/'), following)
+      const targetSegments = target.split('/')
+      if (!targetSegments[targetSegments.length - 1]) targetSegments.pop()
+      return readGitObjectAtPath(repo, root, targetParent, targetSegments, following)
     })
   }
   const err = { name: 'SymbolicLinkCycleError', code: 'SymbolicLinkCycleError', oid }
@@ -688,7 +674,7 @@ function loadComponentDescriptor (files, ref, version) {
       matched = version[refname]
     } else if (
       !Object.entries(version).some(([pattern, replacement]) => {
-        const result = refname.replace(makePicomatchRx(pattern, PICOMATCH_VERSION_OPTS), '\0' + replacement)
+        const result = refname.replace(makeMatcherRx(pattern, VERSION_MATCHER_OPTS), '\0' + replacement)
         if (result === refname) return false
         matched = result.substr(1)
         return true
@@ -918,34 +904,13 @@ function isDirectory (url) {
   return fsp.stat(url).then((stat) => stat.isDirectory(), invariably.false)
 }
 
-/**
- * Removes the specified directory (including all of its contents) or file.
- * Equivalent to fs.promises.rmdir(dir, { recursive: true }) in Node 12.
- */
-function rmdir (dir) {
-  return fsp
-    .readdir(dir, { withFileTypes: true })
-    .then((lst) =>
-      Promise.all(
-        lst.map((it) =>
-          it.isDirectory()
-            ? rmdir(ospath.join(dir, it.name))
-            : fsp.unlink(ospath.join(dir, it.name)).catch((unlinkErr) => {
-              if (unlinkErr.code !== 'ENOENT') throw unlinkErr
-            })
-        )
-      )
-    )
-    .then(() => fsp.rmdir(dir))
-    .catch((err) => {
-      if (err.code === 'ENOENT') return
-      if (err.code === 'ENOTDIR') {
-        return fsp.unlink(dir).catch((unlinkErr) => {
-          if (unlinkErr.code !== 'ENOENT') throw unlinkErr
-        })
-      }
-      throw err
+function symlinkAwareStat (path_) {
+  return fsp.lstat(path_).then((lstat) => {
+    if (!lstat.isSymbolicLink()) return lstat
+    return fsp.stat(path_).catch((statErr) => {
+      throw Object.assign(statErr, { symlink: true })
     })
+  })
 }
 
 function tagsSpecified (sources) {
@@ -1027,6 +992,10 @@ function transformGitCloneError (err, displayUrl) {
   return wrappedErr
 }
 
+function splitRefPatterns (str) {
+  return ~str.indexOf('{') ? str.split(VENTILATED_CSV_RX) : str.split(CSV_RX)
+}
+
 function coerceToString (value) {
   return value == null ? '' : String(value)
 }
@@ -1039,10 +1008,11 @@ function findWorktrees (repo, patterns) {
   if (!patterns.length) return new Map()
   const linkedOnly = patterns[0] === '.' ? !(patterns = patterns.slice(1)) : true
   let worktreesDir
+  const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   return (patterns.length
     ? fsp
       .readdir((worktreesDir = ospath.join(repo.dir, '.git', 'worktrees')))
-      .then((worktreeNames) => matcher(worktreeNames, [...patterns]), invariably.emptyArray)
+      .then((worktreeNames) => filterRefs(worktreeNames, patterns, patternCache), invariably.emptyArray)
       .then((worktreeNames) =>
         worktreeNames.length
           ? Promise.all(

package/lib/constants.js

@@ -4,23 +4,12 @@ module.exports = Object.freeze({
   COMPONENT_DESC_FILENAME: 'antora.yml',
   CONTENT_CACHE_FOLDER: 'content',
   CONTENT_SRC_GLOB: '**/*[!~]',
-  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, removeBOM: false, uniqueBy: (m) => m },
+  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, strict: false, uniqueBy: (m) => m },
   FILE_MODES: { 100644: 0o100666 & ~process.umask(), 100755: 0o100777 & ~process.umask() },
   GIT_CORE: 'antora',
   GIT_OPERATION_LABEL_LENGTH: 8,
   GIT_PROGRESS_PHASES: ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'],
-  PICOMATCH_VERSION_OPTS: {
-    bash: true,
-    debug: false,
-    dot: true,
-    fastpaths: false,
-    lookbehinds: false,
-    noextglob: true,
-    noglobstar: true,
-    nonegate: true,
-    noquantifiers: true,
-    strictSlashes: true,
-  },
+  REF_PATTERN_CACHE_KEY: Symbol('RefPatternCache'),
   SYMLINK_FILE_MODE: '120000',
   VALID_STATE_FILENAME: 'valid',
 })

package/lib/filter-refs.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const { makeMatcherRx, refMatcherOpts: getMatcherOpts, MATCH_ALL_RX } = require('./matcher')
+
+function compileRx (pattern, opts) {
+  if (pattern === '*' || pattern === '**') return MATCH_ALL_RX
+  return pattern.charAt() === '!' // do our own negate
+    ? Object.defineProperty(makeMatcherRx(pattern.substr(1), opts), 'negated', { value: true })
+    : makeMatcherRx(pattern, opts)
+}
+
+function createMatcher (patterns, cache, opts) {
+  const rxs = patterns.map(
+    (pattern) =>
+      cache.get(pattern) || cache.set(pattern, compileRx(pattern, opts || (opts = getMatcherOpts(cache)))).get(pattern)
+  )
+  if (rxs[0].negated) rxs.unshift(MATCH_ALL_RX)
+  return (candidate) => {
+    let matched
+    for (const rx of rxs) {
+      let voteIfMatched = true
+      if (matched) {
+        if (!rx.negated) continue
+        voteIfMatched = false
+      } else if (rx.negated) {
+        continue
+      }
+      if (rx.test(candidate)) matched = voteIfMatched
+    }
+    return matched
+  }
+}
+
+function filterRefs (candidates, patterns, cache = Object.assign(new Map(), { braces: new Map() })) {
+  const isMatch = createMatcher(patterns, cache)
+  return candidates.reduce((accum, candidate) => {
+    if (isMatch(candidate)) accum.push(candidate)
+    return accum
+  }, [])
+}
+
+module.exports = filterRefs

package/lib/git-credential-manager-store.js

@@ -2,17 +2,16 @@
 
 const { homedir } = require('os')
 const expandPath = require('@antora/expand-path-helper')
-const invariably = { void: () => undefined }
 const { promises: fsp } = require('fs')
+const invariably = require('./invariably')
 const ospath = require('path')
 
 class GitCredentialManagerStore {
   configure ({ config, startDir }) {
     this.entries = undefined
+    this.path = undefined
     this.urls = {}
-    if ((this.contents = (config = config || {}).contents) || !config.path) {
-      this.path = undefined
-    } else {
+    if (!(this.contents = (config = config || {}).contents) && config.path) {
       this.path = expandPath(config.path, { dot: startDir })
     }
     return this
@@ -35,14 +34,13 @@ class GitCredentialManagerStore {
           'git',
           'credentials'
         )
-        contentsPromise = fsp
-          .access(homeGitCredentialsPath)
-          .then(() => fsp.readFile(homeGitCredentialsPath, 'utf8'))
-          .catch(() =>
+        contentsPromise = fsp.access(homeGitCredentialsPath).then(
+          () => fsp.readFile(homeGitCredentialsPath, 'utf8'),
+          () =>
             fsp
               .access(xdgConfigGitCredentialsPath)
               .then(() => fsp.readFile(xdgConfigGitCredentialsPath, 'utf8'), invariably.void)
-          )
+        )
       }
       contentsPromise.then((contents) => {
         if (contents) {

package/lib/index.js

@@ -5,7 +5,7 @@
  *
  * Responsible for aggregating the content from multiple repositories and
  * references into a raw aggregate of virtual files that can be organized by a
- * subsequent step in the pipeline.
+ * subsequent step in the generator.
  *
  * @namespace content-aggregator
  */

package/lib/invariably.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }

package/lib/matcher.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const { compile: bracesToGroup, expand: expandBraces } = require('braces')
+const { makeRe: makeMatcherRx } = require('picomatch')
+
+const BASE_OPTS = {
+  bash: true,
+  dot: true,
+  expandRange: (begin, end, step, opts) => bracesToGroup(opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`),
+  fastpaths: false,
+  nobracket: true,
+  noglobstar: true,
+  nonegate: true,
+  noquantifiers: true,
+  regex: false,
+  strictSlashes: true,
+}
+
+module.exports = {
+  MATCH_ALL_RX: { test: () => true },
+  expandBraces,
+  makeMatcherRx,
+  pathMatcherOpts: Object.assign({}, BASE_OPTS, { dot: false }),
+  refMatcherOpts: (cache) =>
+    Object.assign({}, BASE_OPTS, {
+      expandRange: (begin, end, step, opts) => {
+        const pattern = opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`
+        return cache.braces.get(pattern) || cache.braces.set(pattern, bracesToGroup(pattern)).get(pattern)
+      },
+    }),
+  versionMatcherOpts: Object.assign({}, BASE_OPTS, { nonegate: false }),
+}

package/lib/resolve-path-globs.js

@@ -1,26 +1,21 @@
 'use strict'
 
-const { expand: expandBraces } = require('braces')
 const flattenDeep = require('./flatten-deep')
 const { promises: fsp } = require('fs')
 const git = require('./git')
-const invariably = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }
-const { makeRe: makePicomatchRx } = require('picomatch')
+const invariably = require('./invariably')
+const { expandBraces, makeMatcherRx, pathMatcherOpts: MATCHER_OPTS } = require('./matcher')
 
-const RX_ESCAPE_EXCEPT_GLOB = /[.+?^${}()|[\]\\]/g
-const RX_MAGIC_DETECTOR = /[*{]/
-const RX_QUESTION_MARK = /\?/g
-const PICOMATCH_OPTS = { nobracket: true, noextglob: true, noglobstar: true, noquantifiers: true }
-const PICOMATCH_NEGATED_OPTS = { nobracket: true, noextglob: true, noquantifiers: true }
+const NON_GLOB_SPECIAL_CHARS_RX = /[.+?^${}()|[\]\\]/g
+const RX_MAGIC_DETECTOR = /[*{(]/
 
 function resolvePathGlobs (base, patterns, listDirents, retrievePath, tree = { path: '' }) {
   return patterns.reduce((paths, pattern) => {
     if (pattern.charAt() === '!') {
       return paths.then((resolvedPaths) => {
         if (resolvedPaths.length) {
-          if (~pattern.indexOf('?')) pattern = pattern.replace(RX_QUESTION_MARK, '\\?')
-          const rx = makePicomatchRx(pattern, PICOMATCH_NEGATED_OPTS)
-          return resolvedPaths.filter(rx.test.bind(rx))
+          const rx = makeMatcherRx(pattern.substr(1), MATCHER_OPTS)
+          return resolvedPaths.filter((it) => !rx.test(it))
         } else {
           return resolvedPaths
         }
@@ -38,15 +33,14 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
   let patternSegment = patternSegments[0]
   patternSegments = patternSegments.slice(1)
   if (RX_MAGIC_DETECTOR.test(patternSegment)) {
-    let isMatch
-    let explicit
+    let isMatch, explicit
     if (patternSegment === '*') {
       isMatch = (it) => it.charAt() !== '.'
+    } else if (~patternSegment.indexOf('(')) {
+      isMatch = (isMatch = makeMatcherRx(patternSegment, MATCHER_OPTS)).test.bind(isMatch)
     } else if (~patternSegment.indexOf('{')) {
       if (globbed) {
-        if (patternSegment.charAt() === '!') patternSegment = '\\' + patternSegment
-        if (~patternSegment.indexOf('?')) patternSegment = patternSegment.replace(RX_QUESTION_MARK, '\\?')
-        isMatch = (isMatch = makePicomatchRx(patternSegment, PICOMATCH_OPTS)).test.bind(isMatch)
+        isMatch = (isMatch = makeMatcherRx(patternSegment, MATCHER_OPTS)).test.bind(isMatch)
       } else if (~patternSegment.indexOf('*')) {
         const [wildPatterns, literals] = expandBraces(patternSegment).reduce(
           ([wild, literal], it) => (~it.indexOf('*') ? [[...wild, it], literal] : [wild, [...literal, it]]),
@@ -58,7 +52,7 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
         return expandBraces(patternSegment).map((it) => joinPath(path, it))
       }
     } else {
-      isMatch = (isMatch = makeMatcherRx(patternSegment)).test.bind(isMatch)
+      isMatch = (isMatch = makeSingleMatcherRx(patternSegment)).test.bind(isMatch)
     }
     let dirents = await listDirents(base, oid || path)
     if (explicit) dirents = dirents.filter((dirent) => !explicit.has(dirent.name))
@@ -90,22 +84,15 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
         })
       } else if ((patternSegment += '/' + patternSegments.join('/')).indexOf('{')) {
         return expandBraces(patternSegment).map((it) => joinPath(path, it))
-      } else {
-        return [joinPath(path, patternSegment)]
       }
+      return [joinPath(path, patternSegment)]
     } else if (globbed) {
       return (await retrievePath(base, { oid, path }, patternSegment)) ? [joinPath(path, patternSegment)] : []
-    } else {
-      return [joinPath(path, patternSegment)]
     }
+    return [joinPath(path, patternSegment)]
   }
 }
 
-function regexpEscapeWithGlob (str) {
-  // we don't escape "-" since it's meaningless in a literal
-  return str.replace(RX_ESCAPE_EXCEPT_GLOB, '\\$&').replace('*', '.*')
-}
-
 function extractMagicBase (patternSegments, base) {
   let nextSegment
   if (patternSegments.length) {
@@ -135,12 +122,18 @@ function makeAlternationMatcherRx (patterns) {
   return new RegExp('^(?:' + patterns.map(patternToRx).join('|') + ')$')
 }
 
-function makeMatcherRx (pattern) {
+function makeSingleMatcherRx (pattern) {
   return new RegExp('^' + patternToRx(pattern) + '$')
 }
 
 function patternToRx (pattern) {
-  return (pattern.charAt() === '.' ? '' : '(?!\\.)') + regexpEscapeWithGlob(pattern)
+  return (
+    (pattern.charAt() === '.' ? '' : '(?!\\.)') +
+    pattern
+      .replace(NON_GLOB_SPECIAL_CHARS_RX, '\\$&')
+      .replace('\\\\*', '\\x2a')
+      .replace('*', '.*?')
+  )
 }
 
 function readdirWithFileTypes (dir) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.0.0-alpha.9",
+  "version": "3.0.0-beta.4",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -21,21 +21,23 @@
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~6.2",
+    "camelcase-keys": "~7.0",
+    "glob-stream": "~7.0",
     "hpagent": "~0.1.0",
     "isomorphic-git": "~1.10",
     "js-yaml": "~4.1",
-    "matcher": "~4.0",
     "multi-progress": "~4.0",
     "picomatch": "~2.3",
     "progress": "~2.0",
     "should-proxy": "~1.0",
     "simple-get": "~4.0",
-    "vinyl": "~2.2",
-    "vinyl-fs": "~3.0"
+    "vinyl": "~2.2"
+  },
+  "devDependencies": {
+    "node-git-server": "~0.6"
   },
   "engines": {
-    "node": ">=10.17.0"
+    "node": ">=12.21.0"
   },
   "files": [
     "lib/"
@@ -50,5 +52,6 @@
     "static site",
     "web publishing"
   ],
-  "gitHead": "a504d6889819b548e8a5416a7194cbb6f9a93e93"
+  "gitHead": "8a142499e9f1a9e0631777796e06dd6c010d3a90",
+  "readmeFilename": "README.md"
 }

package/lib/promise-all-settled-polyfill.js

@@ -1,11 +0,0 @@
-'use strict'
-
-Promise.allSettled = (iterable) =>
-  Promise.all(
-    iterable.map((it) =>
-      it.then(
-        (value) => ({ status: 'fulfilled', value }),
-        (reason) => ({ status: 'rejected', reason })
-      )
-    )
-  )