@antora/content-aggregator 3.0.0-alpha.8 → 3.0.0-beta.3

package/README.md

@@ -3,7 +3,7 @@
 The Content Aggregator is a component in Antora responsible for fetching and aggregating content distributed across multiple local and remote git repositories for use in an Antora documentation pipeline.
 
 [Antora](https://antora.org) is a modular static site generator designed for creating documentation sites from AsciiDoc documents.
-Its site generator pipeline aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
+Its site generator aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
 
 ## Copyright and License
 

package/lib/aggregate-content.js

@@ -7,6 +7,7 @@ const decodeUint8Array = require('./decode-uint8-array')
 const EventEmitter = require('events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
+const filterRefs = require('./filter-refs')
 const flattenDeep = require('./flatten-deep')
 const fs = require('fs')
 const { promises: fsp } = fs
@@ -14,18 +15,17 @@ const getCacheDir = require('cache-directory')
 const GitCredentialManagerStore = require('./git-credential-manager-store')
 const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
-const invariably = { false: () => false, void: () => undefined, emptyArray: () => [] }
-const { makeRe: makePicomatchRx } = require('picomatch')
-const matcher = require('matcher')
+const globStream = require('glob-stream')
+const invariably = require('./invariably')
+const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')
 const MultiProgress = require('multi-progress')
 const ospath = require('path')
 const { posix: path } = ospath
 const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
 const { fs: resolvePathGlobsFs, git: resolvePathGlobsGit } = require('./resolve-path-globs')
-const { Transform } = require('stream')
-const map = (transform, flush = undefined) => new Transform({ objectMode: true, transform, flush })
+const { pipeline, Transform } = require('stream')
+const map = (transform) => new Transform({ objectMode: true, transform })
 const userRequire = require('@antora/user-require-helper')
-const vfs = require('vinyl-fs')
 const yaml = require('js-yaml')
 
 const {
@@ -37,11 +37,10 @@ const {
   GIT_CORE,
   GIT_OPERATION_LABEL_LENGTH,
   GIT_PROGRESS_PHASES,
-  PICOMATCH_VERSION_OPTS,
+  REF_PATTERN_CACHE_KEY,
   SYMLINK_FILE_MODE,
   VALID_STATE_FILENAME,
 } = require('./constants')
-
 const ANY_SEPARATOR_RX = /[:/]/
 const CSV_RX = /\s*,\s*/
 const VENTILATED_CSV_RX = /\s*,\s+/
@@ -89,48 +88,70 @@ const URL_PORT_CLEANER_RX = /^([^/]+):[0-9]+(?=\/)/
 function aggregateContent (playbook) {
   const startDir = playbook.dir || '.'
   const { branches, editUrl, tags, sources } = playbook.content
-  const { cacheDir, fetch, quiet } = playbook.runtime
-  return ensureCacheDir(cacheDir, startDir).then((resolvedCacheDir) => {
-    const gitPlugins = loadGitPlugins(
-      Object.assign({ ensureGitSuffix: true }, playbook.git),
-      playbook.network || {},
-      startDir
-    )
-    const sourcesByUrl = sources.reduce(
-      (accum, source) => accum.set(source.url, [...(accum.get(source.url) || []), source]),
-      new Map()
-    )
+  const sourceDefaults = { branches, editUrl, tags }
+  const { cacheDir: requestedCacheDir, fetch, quiet } = playbook.runtime
+  return ensureCacheDir(requestedCacheDir, startDir).then((cacheDir) => {
+    const gitConfig = Object.assign({ ensureGitSuffix: true }, playbook.git)
+    const gitPlugins = loadGitPlugins(gitConfig, playbook.network || {}, startDir)
+    const fetchConcurrency = Math.max(gitConfig.fetchConcurrency || Infinity, 1)
+    const sourcesByUrl = sources.reduce((accum, source) => {
+      return accum.set(source.url, [...(accum.get(source.url) || []), Object.assign({}, sourceDefaults, source)])
+    }, new Map())
     const progress = !quiet && createProgress(sourcesByUrl.keys(), process.stdout)
-    return Promise.all(
-      [...sourcesByUrl.entries()].map(([url, sources]) =>
-        loadRepository(url, {
-          cacheDir: resolvedCacheDir,
-          gitPlugins,
-          fetchTags: tagsSpecified(sources, tags),
-          progress,
-          fetch,
-          startDir,
-        }).then(({ repo, authStatus }) =>
-          Promise.all(
-            sources.map((source) => {
-              source = Object.assign({ branches, editUrl, tags }, source)
-              // NOTE if repository is managed (has a url property), we can assume the remote name is origin
-              // TODO if the repo has no remotes, then remoteName should be undefined
-              const remoteName = repo.url ? 'origin' : source.remote || 'origin'
-              return collectFilesFromSource(source, repo, remoteName, authStatus)
-            })
-          )
-        )
-      )
-    )
-      .then(buildAggregate)
-      .catch((err) => {
-        progress && progress.terminate()
-        throw err
-      })
+    const refPatternCache = Object.assign(new Map(), { braces: new Map() })
+    const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir, refPatternCache }
+    return collectFiles(sourcesByUrl, loadOpts, fetchConcurrency).then(buildAggregate, (err) => {
+      progress && progress.terminate()
+      throw err
+    })
   })
 }
 
+async function collectFiles (sourcesByUrl, loadOpts, concurrency) {
+  const tasks = [...sourcesByUrl.entries()].map(([url, sources]) => [
+    () => loadRepository(url, Object.assign({ fetchTags: tagsSpecified(sources) }, loadOpts)),
+    ({ repo, authStatus }) =>
+      Promise.all(
+        sources.map((source) => {
+          // NOTE if repository is managed (has a url property), we can assume the remote name is origin
+          // TODO if the repo has no remotes, then remoteName should be undefined
+          const remoteName = repo.url ? 'origin' : source.remote || 'origin'
+          return collectFilesFromSource(source, repo, remoteName, authStatus)
+        })
+      ),
+  ])
+  let rejection, started
+  const startedContinuations = []
+  const recordRejection = (err) => {
+    rejection = err
+  }
+  const runTask = (primary, continuation, idx) =>
+    primary().then((value) => {
+      if (!rejection) startedContinuations[idx] = continuation(value).catch(recordRejection)
+    }, recordRejection)
+  if (tasks.length > concurrency) {
+    started = []
+    const pending = []
+    for (const [primary, continuation] of tasks) {
+      const current = runTask(primary, continuation, started.length).finally(() =>
+        pending.splice(pending.indexOf(current), 1)
+      )
+      started.push(current)
+      if (pending.push(current) < concurrency) continue
+      await Promise.race(pending)
+      if (rejection) break
+    }
+  } else {
+    started = tasks.map(([primary, continuation], idx) => runTask(primary, continuation, idx))
+  }
+  return Promise.all(started).then(() =>
+    Promise.all(startedContinuations).then((result) => {
+      if (rejection) throw rejection
+      return result
+    })
+  )
+}
+
 function buildAggregate (componentVersionBuckets) {
   return [
     ...flattenDeep(componentVersionBuckets)
@@ -148,17 +169,15 @@ function buildAggregate (componentVersionBuckets) {
 }
 
 async function loadRepository (url, opts) {
-  let dir
-  let repo
-  let authStatus
+  let authStatus, dir, repo
+  const cache = { [REF_PATTERN_CACHE_KEY]: opts.refPatternCache }
   if (~url.indexOf(':') && GIT_URI_DETECTOR_RX.test(url)) {
-    let displayUrl
-    let credentials
+    let credentials, displayUrl
     ;({ displayUrl, url, credentials } = extractCredentials(url))
     const { cacheDir, fetch, fetchTags, gitPlugins, progress } = opts
     dir = ospath.join(cacheDir, generateCloneFolderName(displayUrl))
     // NOTE the presence of the url property on the repo object implies the repository is remote
-    repo = { cache: {}, dir, fs, gitdir: dir, noCheckout: true, url }
+    repo = { cache, dir, fs, gitdir: dir, noCheckout: true, url }
     const validStateFile = ospath.join(dir, VALID_STATE_FILENAME)
     try {
       await fsp.access(validStateFile)
@@ -183,7 +202,7 @@ async function loadRepository (url, opts) {
         authStatus = await git.getConfig(Object.assign({ path: 'remote.origin.private' }, repo))
       }
     } catch (gitErr) {
-      await rmdir(dir)
+      await fsp['rm' in fsp ? 'rm' : 'rmdir'](dir, { recursive: true, force: true })
       if (gitErr.rethrow) throw transformGitCloneError(gitErr, displayUrl)
       const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetchTags, 'clone')
       await git
@@ -194,7 +213,7 @@ async function loadRepository (url, opts) {
           authStatus = credentials ? 'auth-embedded' : credentialManager.status({ url }) ? 'auth-required' : undefined
           return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
         })
-        .catch(async (cloneErr) => {
+        .catch((cloneErr) => {
           // FIXME triggering the error handler here causes assertion problems in the test suite
           //if (fetchOpts.onProgress) fetchOpts.onProgress.finish(cloneErr)
           throw transformGitCloneError(cloneErr, displayUrl)
@@ -204,9 +223,7 @@ async function loadRepository (url, opts) {
     }
   } else if (await isDirectory((dir = expandPath(url, { dot: opts.startDir })))) {
     const gitdir = ospath.join(dir, '.git')
-    repo = (await isDirectory(gitdir))
-      ? { cache: {}, dir, fs, gitdir }
-      : { cache: {}, dir, fs, gitdir: dir, noCheckout: true }
+    repo = (await isDirectory(gitdir)) ? { cache, dir, fs, gitdir } : { cache, dir, fs, gitdir: dir, noCheckout: true }
     try {
       await git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo))
     } catch {
@@ -250,99 +267,102 @@ async function collectFilesFromSource (source, repo, remoteName, authStatus) {
 async function selectReferences (source, repo, remote) {
   let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns = '.' } = source
   const isBare = repo.noCheckout
+  const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   const noWorktree = repo.url ? undefined : null
   const refs = new Map()
-  if (tagPatterns) {
-    tagPatterns = Array.isArray(tagPatterns)
+  if (
+    tagPatterns &&
+    (tagPatterns = Array.isArray(tagPatterns)
       ? tagPatterns.map((pattern) => String(pattern))
-      : String(tagPatterns).split(CSV_RX)
-    if (tagPatterns.length) {
-      const tags = await git.listTags(repo)
-      for (const shortname of tags.length ? matcher(tags, tagPatterns) : tags) {
+      : splitRefPatterns(String(tagPatterns))).length
+  ) {
+    const tags = await git.listTags(repo)
+    if (tags.length) {
+      for (const shortname of filterRefs(tags, tagPatterns, patternCache)) {
         // NOTE tags are stored using symbol keys to distinguish them from branches
         refs.set(Symbol(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
       }
     }
   }
-  if (branchPatterns) {
-    if (worktreePatterns) {
-      if (worktreePatterns === '.') {
-        worktreePatterns = ['.']
-      } else if (worktreePatterns === true) {
-        worktreePatterns = ['.', '*']
-      } else {
-        worktreePatterns = Array.isArray(worktreePatterns)
-          ? worktreePatterns.map((pattern) => String(pattern))
-          : String(worktreePatterns).split(CSV_RX)
-      }
+  if (!branchPatterns) return [...refs.values()]
+  if (worktreePatterns) {
+    if (worktreePatterns === '.') {
+      worktreePatterns = ['.']
+    } else if (worktreePatterns === true) {
+      worktreePatterns = ['.', '*']
+    } else {
+      worktreePatterns = Array.isArray(worktreePatterns)
+        ? worktreePatterns.map((pattern) => String(pattern))
+        : splitRefPatterns(String(worktreePatterns))
+    }
+  }
+  const branchPatternsString = String(branchPatterns)
+  if (branchPatternsString === 'HEAD' || branchPatternsString === '.') {
+    const currentBranch = await getCurrentBranchName(repo, remote)
+    if (currentBranch) {
+      branchPatterns = [currentBranch]
+    } else if (isBare) {
+      return [...refs.values()]
+    } else {
+      // NOTE current branch is undefined when HEAD is detached
+      const head = worktreePatterns[0] === '.' ? repo.dir : noWorktree
+      refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
+      return [...refs.values()]
     }
-    const branchPatternsString = String(branchPatterns)
-    if (branchPatternsString === 'HEAD' || branchPatternsString === '.') {
+  } else if (
+    (branchPatterns = Array.isArray(branchPatterns)
+      ? branchPatterns.map((pattern) => String(pattern))
+      : splitRefPatterns(branchPatternsString)).length
+  ) {
+    let headBranchIdx
+    // NOTE we can assume at least two entries if HEAD or . are present
+    if (~(headBranchIdx = branchPatterns.indexOf('HEAD')) || ~(headBranchIdx = branchPatterns.indexOf('.'))) {
       const currentBranch = await getCurrentBranchName(repo, remote)
       if (currentBranch) {
-        branchPatterns = [currentBranch]
-      } else {
-        if (!isBare) {
-          // NOTE current branch is undefined when HEAD is detached
-          const head = worktreePatterns[0] === '.' ? repo.dir : noWorktree
-          refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
-        }
-        return [...refs.values()]
-      }
-    } else if (
-      (branchPatterns = Array.isArray(branchPatterns)
-        ? branchPatterns.map((pattern) => String(pattern))
-        : branchPatternsString.split(CSV_RX)).length
-    ) {
-      let headBranchIdx
-      // NOTE we can assume at least two entries if HEAD or . are present
-      if (~(headBranchIdx = branchPatterns.indexOf('HEAD')) || ~(headBranchIdx = branchPatterns.indexOf('.'))) {
-        const currentBranch = await getCurrentBranchName(repo, remote)
-        if (currentBranch) {
-          // NOTE ignore if current branch is already in list
-          if (~branchPatterns.indexOf(currentBranch)) {
-            branchPatterns.splice(headBranchIdx, 1)
-          } else {
-            branchPatterns[headBranchIdx] = currentBranch
-          }
-        } else {
-          if (!isBare) {
-            let head = noWorktree
-            if (worktreePatterns[0] === '.') {
-              worktreePatterns = worktreePatterns.slice(1)
-              head = repo.dir
-            }
-            // NOTE current branch is undefined when HEAD is detached
-            refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
-          }
+        // NOTE ignore if current branch is already in list
+        if (~branchPatterns.indexOf(currentBranch)) {
           branchPatterns.splice(headBranchIdx, 1)
+        } else {
+          branchPatterns[headBranchIdx] = currentBranch
         }
+      } else if (isBare) {
+        branchPatterns.splice(headBranchIdx, 1)
+      } else {
+        let head = noWorktree
+        if (worktreePatterns[0] === '.') {
+          worktreePatterns = worktreePatterns.slice(1)
+          head = repo.dir
+        }
+        // NOTE current branch is undefined when HEAD is detached
+        refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
+        branchPatterns.splice(headBranchIdx, 1)
       }
-    } else {
-      return [...refs.values()]
     }
-    // NOTE isomorphic-git includes HEAD in list of remote branches (see https://isomorphic-git.org/docs/listBranches)
-    const remoteBranches = (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
-    if (remoteBranches.length) {
-      for (const shortname of matcher(remoteBranches, branchPatterns)) {
-        const fullname = 'remotes/' + remote + '/' + shortname
-        refs.set(shortname, { shortname, fullname, type: 'branch', remote, head: noWorktree })
-      }
+  } else {
+    return [...refs.values()]
+  }
+  // NOTE isomorphic-git includes HEAD in list of remote branches (see https://isomorphic-git.org/docs/listBranches)
+  const remoteBranches = (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
+  if (remoteBranches.length) {
+    for (const shortname of filterRefs(remoteBranches, branchPatterns, patternCache)) {
+      const fullname = 'remotes/' + remote + '/' + shortname
+      refs.set(shortname, { shortname, fullname, type: 'branch', remote, head: noWorktree })
     }
-    // NOTE only consider local branches if repo has a worktree or there are no remote tracking branches
-    if (!isBare) {
-      const localBranches = await git.listBranches(repo)
-      if (localBranches.length) {
-        const worktrees = await findWorktrees(repo, worktreePatterns)
-        for (const shortname of matcher(localBranches, branchPatterns)) {
-          const head = worktrees.get(shortname) || noWorktree
-          refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
-        }
+  }
+  // NOTE only consider local branches if repo has a worktree or there are no remote tracking branches
+  if (!isBare) {
+    const localBranches = await git.listBranches(repo)
+    if (localBranches.length) {
+      const worktrees = await findWorktrees(repo, worktreePatterns)
+      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
+        const head = worktrees.get(shortname) || noWorktree
+        refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
       }
-    } else if (!remoteBranches.length) {
-      // QUESTION should local branches be used if the only remote branch is HEAD?
-      const localBranches = await git.listBranches(repo)
-      for (const shortname of localBranches.length ? matcher(localBranches, branchPatterns) : localBranches) {
+    }
+  } else if (!remoteBranches.length) {
+    const localBranches = await git.listBranches(repo)
+    if (localBranches.length) {
+      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
         refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head: noWorktree })
       }
     }
@@ -417,68 +437,53 @@ function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePa
 
 function readFilesFromWorktree (worktreePath, startPath) {
   const cwd = ospath.join(worktreePath, startPath)
-  return fsp
-    .stat(cwd)
-    .catch(() => {
+  return fsp.stat(cwd).then(
+    (startPathStat) => {
+      if (!startPathStat.isDirectory()) throw new Error(`the start path '${startPath}' is not a directory`)
+      return srcFs(cwd)
+    },
+    () => {
       throw new Error(`the start path '${startPath}' does not exist`)
-    })
-    .then((stat) => {
-      if (!stat.isDirectory()) throw new Error(`the start path '${startPath}' is not a directory`)
-      return new Promise((resolve, reject) =>
-        vfs
-          .src(CONTENT_SRC_GLOB, Object.assign({ cwd }, CONTENT_SRC_OPTS))
-          .on('error', (err) => {
-            if (err.code === 'ENOENT' && err.syscall === 'stat') {
-              try {
-                if (fs.lstatSync(err.path).isSymbolicLink()) {
-                  err.message = `Broken symbolic link detected at ${ospath.relative(cwd, err.path)}`
-                }
-              } catch {}
-            } else if (err.code === 'ELOOP') {
-              err.message = `Symbolic link cycle detected at ${ospath.relative(cwd, err.path)}`
-            }
-            reject(err)
-          })
-          .pipe(relativizeFiles())
-          .pipe(collectFiles(resolve))
-      )
-    })
-}
-
-/**
- * Transforms the path of every file in the stream to a relative posix path.
- *
- * Applies a mapping function to all files in the stream so they end up with a
- * posixified path relative to the file's base instead of the filesystem root.
- * This mapper also filters out any directories (indicated by file.isNull())
- * that got caught up in the glob.
- */
-function relativizeFiles () {
-  return map((file, enc, next) => {
-    if (file.isNull()) {
-      next()
-    } else {
-      next(
-        null,
-        new File({
-          path: posixify ? posixify(file.relative) : file.relative,
-          contents: file.contents,
-          stat: file.stat,
-          src: { abspath: file.path },
-        })
-      )
     }
-  })
+  )
 }
 
-function collectFiles (done) {
-  const accum = []
-  return map(
-    (file, enc, next) => {
-      accum.push(file)
-      next()
-    },
-    () => done(accum)
+function srcFs (cwd) {
+  return new Promise((resolve, reject, cache = {}, files = []) =>
+    pipeline(
+      globStream(CONTENT_SRC_GLOB, Object.assign({ cache, cwd }, CONTENT_SRC_OPTS)),
+      map(({ path: abspathPosix }, _, next) => {
+        if (Array.isArray(cache[abspathPosix])) return next() // optimization, but not guaranteed
+        const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
+        const relpath = abspath.substr(cwd.length + 1)
+        symlinkAwareStat(abspath).then(
+          (stat) => {
+            if (stat.isDirectory()) return next()
+            fsp.readFile(abspath).then(
+              (contents) => {
+                files.push(new File({ path: posixify ? posixify(relpath) : relpath, contents, stat, src: { abspath } }))
+                next()
+              },
+              (readErr) => {
+                next(Object.assign(readErr, { message: readErr.message.replace(`'${abspath}'`, relpath) }))
+              }
+            )
+          },
+          (statErr) => {
+            if (statErr.symlink) {
+              statErr.message =
+                statErr.code === 'ELOOP'
+                  ? `Symbolic link cycle detected at ${relpath}`
+                  : `Broken symbolic link detected at ${relpath}`
+            } else {
+              statErr.message = statErr.message.replace(`'${abspath}'`, relpath)
+            }
+            next(statErr)
+          }
+        )
+      }),
+      (err) => (err ? reject(err) : resolve(files))
+    )
   )
 }
 
@@ -493,13 +498,13 @@ function readFilesFromGitTree (repo, oid, startPath) {
 }
 
 function getGitTreeAtStartPath (repo, oid, startPath) {
-  return git
-    .readTree(Object.assign({ oid, filepath: startPath }, repo))
-    .catch((err) => {
+  return git.readTree(Object.assign({ oid, filepath: startPath }, repo)).then(
+    (result) => Object.assign(result, { dirname: startPath }),
+    (err) => {
       const m = err instanceof ObjectTypeError && err.data.expected === 'tree' ? 'is not a directory' : 'does not exist'
       throw new Error(`the start path '${startPath}' ${m}`)
-    })
-    .then((result) => Object.assign(result, { dirname: startPath }))
+    }
+  )
 }
 
 function srcGitTree (repo, root, start) {
@@ -518,9 +523,11 @@ function createGitTreeWalker (repo, root, filter) {
     walk (start) {
       return (
         visitGitTree(this, repo, root, filter, start)
-          .then(() => this.emit('end'))
           // NOTE if error is thrown, promises already being resolved won't halt
-          .catch((err) => this.emit('error', err))
+          .then(
+            () => this.emit('end'),
+            (err) => this.emit('error', err)
+          )
       )
     },
   })
@@ -543,8 +550,15 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
         let mode
         if (entry.mode === SYMLINK_FILE_MODE) {
           reads.push(
-            readGitSymlink(repo, root, parent, entry, following)
-              .catch((err) => {
+            readGitSymlink(repo, root, parent, entry, following).then(
+              (target) => {
+                if (target.type === 'tree') {
+                  return visitGitTree(emitter, repo, root, filter, target, vfilePath, new Set(following).add(entry.oid))
+                } else if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
+                  emitter.emit('entry', Object.assign({ mode, oid: target.oid, path: vfilePath }, repo))
+                }
+              },
+              (err) => {
                 // NOTE this error could be caught after promise chain has already been rejected
                 if (err instanceof NotFoundError) {
                   err.message = `Broken symbolic link detected at ${vfilePath}`
@@ -552,14 +566,8 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
                   err.message = `Symbolic link cycle detected at ${vfilePath}`
                 }
                 throw err
-              })
-              .then((target) => {
-                if (target.type === 'tree') {
-                  return visitGitTree(emitter, repo, root, filter, target, vfilePath, new Set(following).add(entry.oid))
-                } else if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
-                  emitter.emit('entry', Object.assign({ mode, oid: target.oid, path: vfilePath }, repo))
-                }
-              })
+              }
+            )
           )
         } else if ((mode = FILE_MODES[entry.mode])) {
           emitter.emit('entry', Object.assign({ mode, oid: entry.oid, path: vfilePath }, repo))
@@ -577,7 +585,7 @@ function readGitSymlink (repo, root, parent, { oid }, following) {
       let targetParent
       if (parent.dirname) {
         const dirname = parent.dirname + '/'
-        target = path.join(dirname, target)
+        target = path.join(dirname, target) // join doesn't remove trailing separator
         if (target.startsWith(dirname)) {
           target = target.substr(dirname.length)
           targetParent = parent
@@ -585,10 +593,12 @@ function readGitSymlink (repo, root, parent, { oid }, following) {
           targetParent = root
         }
       } else {
-        target = path.normalize(target)
+        target = path.normalize(target) // normalize doesn't remove trailing separator
         targetParent = root
       }
-      return readGitObjectAtPath(repo, root, targetParent, target.split('/'), following)
+      const targetSegments = target.split('/')
+      if (!targetSegments[targetSegments.length - 1]) targetSegments.pop()
+      return readGitObjectAtPath(repo, root, targetParent, targetSegments, following)
     })
   }
   const err = { name: 'SymbolicLinkCycleError', code: 'SymbolicLinkCycleError', oid }
@@ -664,7 +674,7 @@ function loadComponentDescriptor (files, ref, version) {
       matched = version[refname]
     } else if (
       !Object.entries(version).some(([pattern, replacement]) => {
-        const result = refname.replace(makePicomatchRx(pattern, PICOMATCH_VERSION_OPTS), '\0' + replacement)
+        const result = refname.replace(makeMatcherRx(pattern, VERSION_MATCHER_OPTS), '\0' + replacement)
         if (result === refname) return false
         matched = result.substr(1)
         return true
@@ -891,47 +901,20 @@ function resolveRemoteUrl (repo, remoteName) {
  * @return {Boolean} A flag indicating whether the URL matches a directory on the local filesystem.
  */
 function isDirectory (url) {
-  return fsp
-    .stat(url)
-    .then((stat) => stat.isDirectory())
-    .catch(invariably.false)
+  return fsp.stat(url).then((stat) => stat.isDirectory(), invariably.false)
 }
 
-/**
- * Removes the specified directory (including all of its contents) or file.
- * Equivalent to fs.promises.rmdir(dir, { recursive: true }) in Node 12.
- */
-function rmdir (dir) {
-  return fsp
-    .readdir(dir, { withFileTypes: true })
-    .then((lst) =>
-      Promise.all(
-        lst.map((it) =>
-          it.isDirectory()
-            ? rmdir(ospath.join(dir, it.name))
-            : fsp.unlink(ospath.join(dir, it.name)).catch((unlinkErr) => {
-              if (unlinkErr.code !== 'ENOENT') throw unlinkErr
-            })
-        )
-      )
-    )
-    .then(() => fsp.rmdir(dir))
-    .catch((err) => {
-      if (err.code === 'ENOENT') return
-      if (err.code === 'ENOTDIR') {
-        return fsp.unlink(dir).catch((unlinkErr) => {
-          if (unlinkErr.code !== 'ENOENT') throw unlinkErr
-        })
-      }
-      throw err
+function symlinkAwareStat (path_) {
+  return fsp.lstat(path_).then((lstat) => {
+    if (!lstat.isSymbolicLink()) return lstat
+    return fsp.stat(path_).catch((statErr) => {
+      throw Object.assign(statErr, { symlink: true })
     })
+  })
 }
 
-function tagsSpecified (sources, defaultTags) {
-  return ~sources.findIndex((source) => {
-    const tags = source.tags || defaultTags || []
-    return Array.isArray(tags) ? tags.length : true
-  })
+function tagsSpecified (sources) {
+  return sources.some(({ tags }) => tags && (Array.isArray(tags) ? tags.length : true))
 }
 
 function loadGitPlugins (gitConfig, networkConfig, startDir) {
@@ -969,17 +952,16 @@ function ensureCacheDir (preferredCacheDir, startDir) {
       ? getCacheDir('antora' + (process.env.NODE_ENV === 'test' ? '-test' : '')) || ospath.resolve('.antora/cache')
       : expandPath(preferredCacheDir, { dot: startDir })
   const cacheDir = ospath.join(baseCacheDir, CONTENT_CACHE_FOLDER)
-  return fsp
-    .mkdir(cacheDir, { recursive: true })
-    .then(() => cacheDir)
-    .catch((err) => {
+  return fsp.mkdir(cacheDir, { recursive: true }).then(
+    () => cacheDir,
+    (err) => {
       throw Object.assign(err, { message: `Failed to create content cache directory: ${cacheDir}; ${err.message}` })
-    })
+    }
+  )
 }
 
 function transformGitCloneError (err, displayUrl) {
-  let wrappedMsg
-  let trimMessage
+  let wrappedMsg, trimMessage
   if (HTTP_ERROR_CODE_RX.test(err.code)) {
     switch (err.data.statusCode) {
       case 401:
@@ -1010,6 +992,10 @@ function transformGitCloneError (err, displayUrl) {
   return wrappedErr
 }
 
+function splitRefPatterns (str) {
+  return ~str.indexOf('{') ? str.split(VENTILATED_CSV_RX) : str.split(CSV_RX)
+}
+
 function coerceToString (value) {
   return value == null ? '' : String(value)
 }
@@ -1022,11 +1008,11 @@ function findWorktrees (repo, patterns) {
   if (!patterns.length) return new Map()
   const linkedOnly = patterns[0] === '.' ? !(patterns = patterns.slice(1)) : true
   let worktreesDir
+  const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   return (patterns.length
     ? fsp
       .readdir((worktreesDir = ospath.join(repo.dir, '.git', 'worktrees')))
-      .catch(invariably.emptyArray)
-      .then((worktreeNames) => matcher(worktreeNames, [...patterns]))
+      .then((worktreeNames) => filterRefs(worktreeNames, patterns, patternCache), invariably.emptyArray)
       .then((worktreeNames) =>
         worktreeNames.length
           ? Promise.all(

package/lib/constants.js

@@ -2,25 +2,14 @@
 
 module.exports = Object.freeze({
   COMPONENT_DESC_FILENAME: 'antora.yml',
-  CONTENT_CACHE_FOLDER: 'content/2',
+  CONTENT_CACHE_FOLDER: 'content',
   CONTENT_SRC_GLOB: '**/*[!~]',
-  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, removeBOM: false, uniqueBy: (m) => m },
+  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, strict: false, uniqueBy: (m) => m },
   FILE_MODES: { 100644: 0o100666 & ~process.umask(), 100755: 0o100777 & ~process.umask() },
   GIT_CORE: 'antora',
   GIT_OPERATION_LABEL_LENGTH: 8,
   GIT_PROGRESS_PHASES: ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'],
-  PICOMATCH_VERSION_OPTS: {
-    bash: true,
-    debug: false,
-    dot: true,
-    fastpaths: false,
-    lookbehinds: false,
-    noextglob: true,
-    noglobstar: true,
-    nonegate: true,
-    noquantifiers: true,
-    strictSlashes: true,
-  },
+  REF_PATTERN_CACHE_KEY: Symbol('RefPatternCache'),
   SYMLINK_FILE_MODE: '120000',
   VALID_STATE_FILENAME: 'valid',
 })

package/lib/filter-refs.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const { makeMatcherRx, refMatcherOpts: getMatcherOpts, MATCH_ALL_RX } = require('./matcher')
+
+function compileRx (pattern, opts) {
+  if (pattern === '*' || pattern === '**') return MATCH_ALL_RX
+  return pattern.charAt() === '!' // do our own negate
+    ? Object.defineProperty(makeMatcherRx(pattern.substr(1), opts), 'negated', { value: true })
+    : makeMatcherRx(pattern, opts)
+}
+
+function createMatcher (patterns, cache, opts) {
+  const rxs = patterns.map(
+    (pattern) =>
+      cache.get(pattern) || cache.set(pattern, compileRx(pattern, opts || (opts = getMatcherOpts(cache)))).get(pattern)
+  )
+  if (rxs[0].negated) rxs.unshift(MATCH_ALL_RX)
+  return (candidate) => {
+    let matched
+    for (const rx of rxs) {
+      let voteIfMatched = true
+      if (matched) {
+        if (!rx.negated) continue
+        voteIfMatched = false
+      } else if (rx.negated) {
+        continue
+      }
+      if (rx.test(candidate)) matched = voteIfMatched
+    }
+    return matched
+  }
+}
+
+function filterRefs (candidates, patterns, cache = Object.assign(new Map(), { braces: new Map() })) {
+  const isMatch = createMatcher(patterns, cache)
+  return candidates.reduce((accum, candidate) => {
+    if (isMatch(candidate)) accum.push(candidate)
+    return accum
+  }, [])
+}
+
+module.exports = filterRefs

package/lib/git-credential-manager-store.js

@@ -3,18 +3,17 @@
 const { homedir } = require('os')
 const expandPath = require('@antora/expand-path-helper')
 const { promises: fsp } = require('fs')
+const invariably = require('./invariably')
 const ospath = require('path')
 
 class GitCredentialManagerStore {
   configure ({ config, startDir }) {
     this.entries = undefined
     this.urls = {}
-    if ((this.contents = (config = config || {}).contents)) {
+    if ((this.contents = (config = config || {}).contents) || !config.path) {
       this.path = undefined
-    } else if (config.path) {
-      this.path = expandPath(config.path, { dot: startDir })
     } else {
-      this.path = undefined
+      this.path = expandPath(config.path, { dot: startDir })
     }
     return this
   }
@@ -28,10 +27,7 @@ class GitCredentialManagerStore {
         contentsPromise = Promise.resolve(this.contents)
         delimiter = /[,\n]/
       } else if (this.path) {
-        contentsPromise = fsp
-          .access(this.path)
-          .then(() => fsp.readFile(this.path, 'utf8'))
-          .catch(() => undefined)
+        contentsPromise = fsp.access(this.path).then(() => fsp.readFile(this.path, 'utf8'), invariably.void)
       } else {
         const homeGitCredentialsPath = ospath.join(homedir(), '.git-credentials')
         const xdgConfigGitCredentialsPath = ospath.join(
@@ -45,8 +41,7 @@ class GitCredentialManagerStore {
           .catch(() =>
             fsp
               .access(xdgConfigGitCredentialsPath)
-              .then(() => fsp.readFile(xdgConfigGitCredentialsPath, 'utf8'))
-              .catch(() => undefined)
+              .then(() => fsp.readFile(xdgConfigGitCredentialsPath, 'utf8'), invariably.void)
           )
       }
       contentsPromise.then((contents) => {

package/lib/git-plugin-http.js

@@ -26,7 +26,7 @@ async function mergeBuffers (data) {
 }
 
 module.exports = ({ httpProxy, httpsProxy, noProxy }, userAgent) => {
-  if (httpsProxy || httpProxy) {
+  if ((httpsProxy || httpProxy) && noProxy !== '*') {
     const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent')
     const shouldProxy = require('should-proxy')
     return {

package/lib/index.js

@@ -5,7 +5,7 @@
  *
  * Responsible for aggregating the content from multiple repositories and
  * references into a raw aggregate of virtual files that can be organized by a
- * subsequent step in the pipeline.
+ * subsequent step in the generator.
  *
  * @namespace content-aggregator
  */

package/lib/invariably.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }

package/lib/matcher.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const { compile: bracesToGroup, expand: expandBraces } = require('braces')
+const { makeRe: makeMatcherRx } = require('picomatch')
+
+const BASE_OPTS = {
+  bash: true,
+  dot: true,
+  expandRange: (begin, end, step, opts) => bracesToGroup(opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`),
+  fastpaths: false,
+  nobracket: true,
+  noglobstar: true,
+  nonegate: true,
+  noquantifiers: true,
+  regex: false,
+  strictSlashes: true,
+}
+
+module.exports = {
+  MATCH_ALL_RX: { test: () => true },
+  expandBraces,
+  makeMatcherRx,
+  pathMatcherOpts: Object.assign({}, BASE_OPTS, { dot: false }),
+  refMatcherOpts: (cache) =>
+    Object.assign({}, BASE_OPTS, {
+      expandRange: (begin, end, step, opts) => {
+        const pattern = opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`
+        return cache.braces.get(pattern) || cache.braces.set(pattern, bracesToGroup(pattern)).get(pattern)
+      },
+    }),
+  versionMatcherOpts: Object.assign({}, BASE_OPTS, { nonegate: false }),
+}

package/lib/resolve-path-globs.js

@@ -1,26 +1,21 @@
 'use strict'
 
-const { expand: expandBraces } = require('braces')
 const flattenDeep = require('./flatten-deep')
 const { promises: fsp } = require('fs')
 const git = require('./git')
-const invariably = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }
-const { makeRe: makePicomatchRx } = require('picomatch')
+const invariably = require('./invariably')
+const { expandBraces, makeMatcherRx, pathMatcherOpts: MATCHER_OPTS } = require('./matcher')
 
-const RX_ESCAPE_EXCEPT_GLOB = /[.+?^${}()|[\]\\]/g
-const RX_MAGIC_DETECTOR = /[*{]/
-const RX_QUESTION_MARK = /\?/g
-const PICOMATCH_OPTS = { nobracket: true, noextglob: true, noglobstar: true, noquantifiers: true }
-const PICOMATCH_NEGATED_OPTS = { nobracket: true, noextglob: true, noquantifiers: true }
+const NON_GLOB_SPECIAL_CHARS_RX = /[.+?^${}()|[\]\\]/g
+const RX_MAGIC_DETECTOR = /[*{(]/
 
 function resolvePathGlobs (base, patterns, listDirents, retrievePath, tree = { path: '' }) {
   return patterns.reduce((paths, pattern) => {
     if (pattern.charAt() === '!') {
       return paths.then((resolvedPaths) => {
         if (resolvedPaths.length) {
-          if (~pattern.indexOf('?')) pattern = pattern.replace(RX_QUESTION_MARK, '\\?')
-          const rx = makePicomatchRx(pattern, PICOMATCH_NEGATED_OPTS)
-          return resolvedPaths.filter(rx.test.bind(rx))
+          const rx = makeMatcherRx(pattern.substr(1), MATCHER_OPTS)
+          return resolvedPaths.filter((it) => !rx.test(it))
         } else {
           return resolvedPaths
         }
@@ -38,15 +33,14 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
   let patternSegment = patternSegments[0]
   patternSegments = patternSegments.slice(1)
   if (RX_MAGIC_DETECTOR.test(patternSegment)) {
-    let isMatch
-    let explicit
+    let isMatch, explicit
     if (patternSegment === '*') {
       isMatch = (it) => it.charAt() !== '.'
+    } else if (~patternSegment.indexOf('(')) {
+      isMatch = (isMatch = makeMatcherRx(patternSegment, MATCHER_OPTS)).test.bind(isMatch)
     } else if (~patternSegment.indexOf('{')) {
       if (globbed) {
-        if (patternSegment.charAt() === '!') patternSegment = '\\' + patternSegment
-        if (~patternSegment.indexOf('?')) patternSegment = patternSegment.replace(RX_QUESTION_MARK, '\\?')
-        isMatch = (isMatch = makePicomatchRx(patternSegment, PICOMATCH_OPTS)).test.bind(isMatch)
+        isMatch = (isMatch = makeMatcherRx(patternSegment, MATCHER_OPTS)).test.bind(isMatch)
       } else if (~patternSegment.indexOf('*')) {
         const [wildPatterns, literals] = expandBraces(patternSegment).reduce(
           ([wild, literal], it) => (~it.indexOf('*') ? [[...wild, it], literal] : [wild, [...literal, it]]),
@@ -58,7 +52,7 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
         return expandBraces(patternSegment).map((it) => joinPath(path, it))
       }
     } else {
-      isMatch = (isMatch = makeMatcherRx(patternSegment)).test.bind(isMatch)
+      isMatch = (isMatch = makeSingleMatcherRx(patternSegment)).test.bind(isMatch)
     }
     let dirents = await listDirents(base, oid || path)
     if (explicit) dirents = dirents.filter((dirent) => !explicit.has(dirent.name))
@@ -90,22 +84,15 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
         })
       } else if ((patternSegment += '/' + patternSegments.join('/')).indexOf('{')) {
         return expandBraces(patternSegment).map((it) => joinPath(path, it))
-      } else {
-        return [joinPath(path, patternSegment)]
       }
+      return [joinPath(path, patternSegment)]
     } else if (globbed) {
       return (await retrievePath(base, { oid, path }, patternSegment)) ? [joinPath(path, patternSegment)] : []
-    } else {
-      return [joinPath(path, patternSegment)]
     }
+    return [joinPath(path, patternSegment)]
   }
 }
 
-function regexpEscapeWithGlob (str) {
-  // we don't escape "-" since it's meaningless in a literal
-  return str.replace(RX_ESCAPE_EXCEPT_GLOB, '\\$&').replace('*', '.*')
-}
-
 function extractMagicBase (patternSegments, base) {
   let nextSegment
   if (patternSegments.length) {
@@ -124,22 +111,29 @@ function listDirentsFs (base, path) {
 function listDirentsGit (repo, treeOid) {
   return git
     .readTree(Object.assign({ oid: treeOid, filepath: '' }, repo))
-    .then(({ tree: entries }) =>
-      entries.map(({ type, oid, path: name }) => ({ name, oid, isDirectory: invariably[type === 'tree'] }))
+    .then(
+      ({ tree: entries }) =>
+        entries.map(({ type, oid, path: name }) => ({ name, oid, isDirectory: invariably[type === 'tree'] })),
+      invariably.emptyArray
     )
-    .catch(invariably.emptyArray)
 }
 
 function makeAlternationMatcherRx (patterns) {
   return new RegExp('^(?:' + patterns.map(patternToRx).join('|') + ')$')
 }
 
-function makeMatcherRx (pattern) {
+function makeSingleMatcherRx (pattern) {
   return new RegExp('^' + patternToRx(pattern) + '$')
 }
 
 function patternToRx (pattern) {
-  return (pattern.charAt() === '.' ? '' : '(?!\\.)') + regexpEscapeWithGlob(pattern)
+  return (
+    (pattern.charAt() === '.' ? '' : '(?!\\.)') +
+    pattern
+      .replace(NON_GLOB_SPECIAL_CHARS_RX, '\\$&')
+      .replace('\\\\*', '\\x2a')
+      .replace('*', '.*?')
+  )
 }
 
 function readdirWithFileTypes (dir) {
@@ -147,10 +141,7 @@ function readdirWithFileTypes (dir) {
 }
 
 function retrievePathFs (base, { path }, subpath) {
-  return fsp
-    .access(base + '/' + joinPath(path, subpath))
-    .then(invariably.true)
-    .catch(invariably.false)
+  return fsp.access(base + '/' + joinPath(path, subpath)).then(invariably.true, invariably.false)
 }
 
 function retrievePathGit (repo, { oid }, filepath) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.0.0-alpha.8",
+  "version": "3.0.0-beta.3",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -21,21 +21,19 @@
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~6.2",
+    "camelcase-keys": "~7.0",
     "hpagent": "~0.1.0",
-    "isomorphic-git": "~1.9",
+    "isomorphic-git": "~1.10",
     "js-yaml": "~4.1",
-    "matcher": "~4.0",
     "multi-progress": "~4.0",
     "picomatch": "~2.3",
     "progress": "~2.0",
     "should-proxy": "~1.0",
     "simple-get": "~4.0",
-    "vinyl": "~2.2",
-    "vinyl-fs": "~3.0"
+    "vinyl": "~2.2"
   },
   "engines": {
-    "node": ">=10.17.0"
+    "node": ">=12.21.0"
   },
   "files": [
     "lib/"
@@ -50,5 +48,5 @@
     "static site",
     "web publishing"
   ],
-  "gitHead": "2e5695bea11fb5719989c329c97e66d36e29659f"
+  "gitHead": "45da95a2e2dea538379d2d9f42013d2208fb86c3"
 }