@antora/content-aggregator 3.0.0-alpha.7 → 3.0.0-beta.2

package/README.md

@@ -3,7 +3,7 @@
 The Content Aggregator is a component in Antora responsible for fetching and aggregating content distributed across multiple local and remote git repositories for use in an Antora documentation pipeline.
 
 [Antora](https://antora.org) is a modular static site generator designed for creating documentation sites from AsciiDoc documents.
-Its site generator pipeline aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
+Its site generator aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
 
 ## Copyright and License
 

package/lib/aggregate-content.js

@@ -3,20 +3,20 @@
 const camelCaseKeys = require('camelcase-keys')
 const { createHash } = require('crypto')
 const createHttpPlugin = require('./git-plugin-http')
-const decodeUint8Data = require('./decode-uint8-data')
+const decodeUint8Array = require('./decode-uint8-array')
 const EventEmitter = require('events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
+const filterRefs = require('./filter-refs')
 const flattenDeep = require('./flatten-deep')
 const fs = require('fs')
 const { promises: fsp } = fs
 const getCacheDir = require('cache-directory')
 const GitCredentialManagerStore = require('./git-credential-manager-store')
-const git = require('isomorphic-git')
+const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
 const invariably = { false: () => false, void: () => undefined, emptyArray: () => [] }
 const { makeRe: makePicomatchRx } = require('picomatch')
-const matcher = require('matcher')
 const MultiProgress = require('multi-progress')
 const ospath = require('path')
 const { posix: path } = ospath
@@ -24,18 +24,21 @@ const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
 const { fs: resolvePathGlobsFs, git: resolvePathGlobsGit } = require('./resolve-path-globs')
 const { Transform } = require('stream')
 const map = (transform, flush = undefined) => new Transform({ objectMode: true, transform, flush })
+const userRequire = require('@antora/user-require-helper')
 const vfs = require('vinyl-fs')
 const yaml = require('js-yaml')
 
 const {
   COMPONENT_DESC_FILENAME,
   CONTENT_CACHE_FOLDER,
-  CONTENT_GLOB,
+  CONTENT_SRC_GLOB,
+  CONTENT_SRC_OPTS,
   FILE_MODES,
   GIT_CORE,
   GIT_OPERATION_LABEL_LENGTH,
   GIT_PROGRESS_PHASES,
   PICOMATCH_VERSION_OPTS,
+  REF_PATTERN_CACHE_KEY,
   SYMLINK_FILE_MODE,
   VALID_STATE_FILENAME,
 } = require('./constants')
@@ -55,6 +58,7 @@ const SPACE_RX = / /g
 const SUPERFLUOUS_SEPARATORS_RX = /^\/+|\/+$|\/+(?=\/)/g
 const URL_AUTH_CLEANER_RX = /^(https?:\/\/)[^/@]*@/
 const URL_AUTH_EXTRACTOR_RX = /^(https?:\/\/)(?:([^/:@]+)?(?::([^/@]+)?)?@)?(.*)/
+const URL_PORT_CLEANER_RX = /^([^/]+):[0-9]+(?=\/)/
 
 /**
  * Aggregates files from the specified content sources so they can be loaded
@@ -86,69 +90,96 @@ const URL_AUTH_EXTRACTOR_RX = /^(https?:\/\/)(?:([^/:@]+)?(?::([^/@]+)?)?@)?(.*)
 function aggregateContent (playbook) {
   const startDir = playbook.dir || '.'
   const { branches, editUrl, tags, sources } = playbook.content
-  const { cacheDir, fetch, quiet } = playbook.runtime
-  return ensureCacheDir(cacheDir, startDir).then((resolvedCacheDir) => {
-    const gitPlugins = loadGitPlugins(
-      Object.assign({ ensureGitSuffix: true }, playbook.git),
-      playbook.network || {},
-      startDir
-    )
-    const sourcesByUrl = sources.reduce(
-      (accum, source) => accum.set(source.url, [...(accum.get(source.url) || []), source]),
-      new Map()
-    )
+  const sourceDefaults = { branches, editUrl, tags }
+  const { cacheDir: requestedCacheDir, fetch, quiet } = playbook.runtime
+  return ensureCacheDir(requestedCacheDir, startDir).then((cacheDir) => {
+    const gitConfig = Object.assign({ ensureGitSuffix: true }, playbook.git)
+    const gitPlugins = loadGitPlugins(gitConfig, playbook.network || {}, startDir)
+    const fetchConcurrency = Math.max(gitConfig.fetchConcurrency || Infinity, 1)
+    const sourcesByUrl = sources.reduce((accum, source) => {
+      return accum.set(source.url, [...(accum.get(source.url) || []), Object.assign({}, sourceDefaults, source)])
+    }, new Map())
     const progress = !quiet && createProgress(sourcesByUrl.keys(), process.stdout)
-    return Promise.all(
-      [...sourcesByUrl.entries()].map(([url, sources]) =>
-        loadRepository(url, {
-          cacheDir: resolvedCacheDir,
-          gitPlugins,
-          fetchTags: tagsSpecified(sources, tags),
-          progress,
-          fetch,
-          startDir,
-        }).then(({ repo, authStatus }) =>
-          Promise.all(
-            sources.map((source) => {
-              source = Object.assign({ branches, editUrl, tags }, source)
-              // NOTE if repository is managed (has a url), we can assume the remote name is origin
-              // TODO if the repo has no remotes, then remoteName should be undefined
-              const remoteName = repo.url ? 'origin' : source.remote || 'origin'
-              return collectFilesFromSource(source, repo, remoteName, authStatus)
-            })
-          )
-        )
-      )
-    )
-      .then(buildAggregate)
-      .catch((err) => {
-        progress && progress.terminate()
-        throw err
-      })
+    const refPatternCache = Object.assign(new Map(), { braces: new Map() })
+    const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir, refPatternCache }
+    return collectFiles(sourcesByUrl, loadOpts, fetchConcurrency).then(buildAggregate, (err) => {
+      progress && progress.terminate()
+      throw err
+    })
   })
 }
 
+async function collectFiles (sourcesByUrl, loadOpts, concurrency) {
+  const tasks = [...sourcesByUrl.entries()].map(([url, sources]) => [
+    () => loadRepository(url, Object.assign({ fetchTags: tagsSpecified(sources) }, loadOpts)),
+    ({ repo, authStatus }) =>
+      Promise.all(
+        sources.map((source) => {
+          // NOTE if repository is managed (has a url property), we can assume the remote name is origin
+          // TODO if the repo has no remotes, then remoteName should be undefined
+          const remoteName = repo.url ? 'origin' : source.remote || 'origin'
+          return collectFilesFromSource(source, repo, remoteName, authStatus)
+        })
+      ),
+  ])
+  let rejection, started
+  const startedContinuations = []
+  const recordRejection = (err) => {
+    rejection = err
+  }
+  const runTask = (primary, continuation, idx) =>
+    primary().then((value) => {
+      if (!rejection) startedContinuations[idx] = continuation(value).catch(recordRejection)
+    }, recordRejection)
+  if (tasks.length > concurrency) {
+    started = []
+    const pending = []
+    for (const [primary, continuation] of tasks) {
+      const current = runTask(primary, continuation, started.length).finally(() =>
+        pending.splice(pending.indexOf(current), 1)
+      )
+      started.push(current)
+      if (pending.push(current) < concurrency) continue
+      await Promise.race(pending)
+      if (rejection) break
+    }
+  } else {
+    started = tasks.map(([primary, continuation], idx) => runTask(primary, continuation, idx))
+  }
+  return Promise.all(started).then(() =>
+    Promise.all(startedContinuations).then((result) => {
+      if (rejection) throw rejection
+      return result
+    })
+  )
+}
+
 function buildAggregate (componentVersionBuckets) {
-  const aggregateMap = flattenDeep(componentVersionBuckets).reduce((accum, batch) => {
-    const key = batch.version + '@' + batch.name
-    const entry = accum.get(key)
-    return accum.set(key, entry ? Object.assign(entry, batch, { files: [...entry.files, ...batch.files] }) : batch)
-  }, new Map())
-  return [...aggregateMap.values()]
+  return [
+    ...flattenDeep(componentVersionBuckets)
+      .reduce((accum, batch) => {
+        const key = batch.version + '@' + batch.name
+        const entry = accum.get(key)
+        if (!entry) return accum.set(key, batch)
+        const files = batch.files
+        ;(batch.files = entry.files).push(...files)
+        Object.assign(entry, batch)
+        return accum
+      }, new Map())
+      .values(),
+  ]
 }
 
 async function loadRepository (url, opts) {
-  let dir
-  let repo
-  let authStatus
+  let authStatus, dir, repo
+  const cache = { [REF_PATTERN_CACHE_KEY]: opts.refPatternCache }
   if (~url.indexOf(':') && GIT_URI_DETECTOR_RX.test(url)) {
-    let displayUrl
-    let credentials
+    let credentials, displayUrl
     ;({ displayUrl, url, credentials } = extractCredentials(url))
     const { cacheDir, fetch, fetchTags, gitPlugins, progress } = opts
     dir = ospath.join(cacheDir, generateCloneFolderName(displayUrl))
     // NOTE the presence of the url property on the repo object implies the repository is remote
-    repo = { cache: {}, dir, fs, gitdir: dir, noCheckout: true, url }
+    repo = { cache, dir, fs, gitdir: dir, noCheckout: true, url }
     const validStateFile = ospath.join(dir, VALID_STATE_FILENAME)
     try {
       await fsp.access(validStateFile)
@@ -173,7 +204,7 @@ async function loadRepository (url, opts) {
         authStatus = await git.getConfig(Object.assign({ path: 'remote.origin.private' }, repo))
       }
     } catch (gitErr) {
-      await rmdir(dir)
+      await fsp['rm' in fsp ? 'rm' : 'rmdir'](dir, { recursive: true, force: true })
       if (gitErr.rethrow) throw transformGitCloneError(gitErr, displayUrl)
       const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetchTags, 'clone')
       await git
@@ -184,8 +215,7 @@ async function loadRepository (url, opts) {
           authStatus = credentials ? 'auth-embedded' : credentialManager.status({ url }) ? 'auth-required' : undefined
           return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
         })
-        .catch(async (cloneErr) => {
-          await rmdir(dir)
+        .catch((cloneErr) => {
           // FIXME triggering the error handler here causes assertion problems in the test suite
           //if (fetchOpts.onProgress) fetchOpts.onProgress.finish(cloneErr)
           throw transformGitCloneError(cloneErr, displayUrl)
@@ -193,10 +223,9 @@ async function loadRepository (url, opts) {
         .then(() => fsp.writeFile(validStateFile, '').catch(invariably.void))
         .then(() => fetchOpts.onProgress && fetchOpts.onProgress.finish())
     }
-  } else if (await isDirectory((dir = expandPath(url, '~+', opts.startDir)))) {
-    repo = (await isDirectory(ospath.join(dir, '.git')))
-      ? { cache: {}, dir, fs }
-      : { cache: {}, dir, fs, gitdir: dir, noCheckout: true }
+  } else if (await isDirectory((dir = expandPath(url, { dot: opts.startDir })))) {
+    const gitdir = ospath.join(dir, '.git')
+    repo = (await isDirectory(gitdir)) ? { cache, dir, fs, gitdir } : { cache, dir, fs, gitdir: dir, noCheckout: true }
     try {
       await git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo))
     } catch {
@@ -240,16 +269,18 @@ async function collectFilesFromSource (source, repo, remoteName, authStatus) {
 async function selectReferences (source, repo, remote) {
   let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns = '.' } = source
   const isBare = repo.noCheckout
+  const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
+  const noWorktree = repo.url ? undefined : null
   const refs = new Map()
   if (tagPatterns) {
     tagPatterns = Array.isArray(tagPatterns)
       ? tagPatterns.map((pattern) => String(pattern))
-      : String(tagPatterns).split(CSV_RX)
+      : splitRefPatterns(String(tagPatterns))
     if (tagPatterns.length) {
       const tags = await git.listTags(repo)
-      for (const shortname of tags.length ? matcher(tags, tagPatterns) : tags) {
+      for (const shortname of tags.length ? filterRefs(tags, tagPatterns, patternCache) : tags) {
         // NOTE tags are stored using symbol keys to distinguish them from branches
-        refs.set(Symbol(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag' })
+        refs.set(Symbol(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
       }
     }
   }
@@ -262,7 +293,7 @@ async function selectReferences (source, repo, remote) {
       } else {
         worktreePatterns = Array.isArray(worktreePatterns)
           ? worktreePatterns.map((pattern) => String(pattern))
-          : String(worktreePatterns).split(CSV_RX)
+          : splitRefPatterns(String(worktreePatterns))
       }
     }
     const branchPatternsString = String(branchPatterns)
@@ -273,16 +304,15 @@ async function selectReferences (source, repo, remote) {
       } else {
         if (!isBare) {
           // NOTE current branch is undefined when HEAD is detached
-          const ref = { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true }
-          if (worktreePatterns[0] === '.') ref.head = repo.dir
-          refs.set('HEAD', ref)
+          const head = worktreePatterns[0] === '.' ? repo.dir : noWorktree
+          refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
         }
         return [...refs.values()]
       }
     } else if (
       (branchPatterns = Array.isArray(branchPatterns)
         ? branchPatterns.map((pattern) => String(pattern))
-        : branchPatternsString.split(CSV_RX)).length
+        : splitRefPatterns(branchPatternsString)).length
     ) {
       let headBranchIdx
       // NOTE we can assume at least two entries if HEAD or . are present
@@ -297,10 +327,13 @@ async function selectReferences (source, repo, remote) {
           }
         } else {
           if (!isBare) {
+            let head = noWorktree
+            if (worktreePatterns[0] === '.') {
+              worktreePatterns = worktreePatterns.slice(1)
+              head = repo.dir
+            }
             // NOTE current branch is undefined when HEAD is detached
-            const ref = { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true }
-            if (worktreePatterns[0] === '.' && (worktreePatterns = worktreePatterns.slice(1))) ref.head = repo.dir
-            refs.set('HEAD', ref)
+            refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
           }
           branchPatterns.splice(headBranchIdx, 1)
         }
@@ -311,8 +344,9 @@ async function selectReferences (source, repo, remote) {
     // NOTE isomorphic-git includes HEAD in list of remote branches (see https://isomorphic-git.org/docs/listBranches)
     const remoteBranches = (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
     if (remoteBranches.length) {
-      for (const shortname of matcher(remoteBranches, branchPatterns)) {
-        refs.set(shortname, { shortname, fullname: path.join('remotes', remote, shortname), type: 'branch', remote })
+      for (const shortname of filterRefs(remoteBranches, branchPatterns, patternCache)) {
+        const fullname = 'remotes/' + remote + '/' + shortname
+        refs.set(shortname, { shortname, fullname, type: 'branch', remote, head: noWorktree })
       }
     }
     // NOTE only consider local branches if repo has a worktree or there are no remote tracking branches
@@ -320,17 +354,17 @@ async function selectReferences (source, repo, remote) {
       const localBranches = await git.listBranches(repo)
       if (localBranches.length) {
         const worktrees = await findWorktrees(repo, worktreePatterns)
-        for (const shortname of matcher(localBranches, branchPatterns)) {
-          const ref = { shortname, fullname: 'heads/' + shortname, type: 'branch' }
-          if (worktrees.has(shortname)) ref.head = worktrees.get(shortname)
-          refs.set(shortname, ref)
+        for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
+          const head = worktrees.get(shortname) || noWorktree
+          refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
         }
       }
     } else if (!remoteBranches.length) {
-      // QUESTION should local branches be used if the only remote branch is HEAD?
       const localBranches = await git.listBranches(repo)
-      for (const shortname of localBranches.length ? matcher(localBranches, branchPatterns) : localBranches) {
-        refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch' })
+      if (localBranches.length) {
+        for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
+          refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head: noWorktree })
+        }
       }
     }
   }
@@ -391,7 +425,7 @@ function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePa
   )
     .then((files) => {
       const componentVersionBucket = loadComponentDescriptor(files, ref, version)
-      const origin = computeOrigin(originUrl, authStatus, ref, startPath, worktreePath, editUrl)
+      const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
       componentVersionBucket.files = files.map((file) => assignFileProperties(file, origin))
       return componentVersionBucket
     })
@@ -404,16 +438,12 @@ function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePa
 
 function readFilesFromWorktree (worktreePath, startPath) {
   const cwd = ospath.join(worktreePath, startPath)
-  return fsp
-    .stat(cwd)
-    .catch(() => {
-      throw new Error(`the start path '${startPath}' does not exist`)
-    })
-    .then((stat) => {
+  return fsp.stat(cwd).then(
+    (stat) => {
       if (!stat.isDirectory()) throw new Error(`the start path '${startPath}' is not a directory`)
       return new Promise((resolve, reject) =>
         vfs
-          .src(CONTENT_GLOB, { cwd, follow: true, removeBOM: false })
+          .src(CONTENT_SRC_GLOB, Object.assign({ cwd }, CONTENT_SRC_OPTS))
           .on('error', (err) => {
             if (err.code === 'ENOENT' && err.syscall === 'stat') {
               try {
@@ -427,9 +457,13 @@ function readFilesFromWorktree (worktreePath, startPath) {
             reject(err)
           })
           .pipe(relativizeFiles())
-          .pipe(collectFiles(resolve))
+          .pipe(collectDataFromStream(resolve))
       )
-    })
+    },
+    () => {
+      throw new Error(`the start path '${startPath}' does not exist`)
+    }
+  )
 }
 
 /**
@@ -458,11 +492,11 @@ function relativizeFiles () {
   })
 }
 
-function collectFiles (done) {
+function collectDataFromStream (done) {
   const accum = []
   return map(
-    (file, enc, next) => {
-      accum.push(file)
+    (obj, _, next) => {
+      accum.push(obj)
       next()
     },
     () => done(accum)
@@ -480,13 +514,13 @@ function readFilesFromGitTree (repo, oid, startPath) {
 }
 
 function getGitTreeAtStartPath (repo, oid, startPath) {
-  return git
-    .readTree(Object.assign({ oid, filepath: startPath }, repo))
-    .catch((err) => {
+  return git.readTree(Object.assign({ oid, filepath: startPath }, repo)).then(
+    (result) => Object.assign(result, { dirname: startPath }),
+    (err) => {
       const m = err instanceof ObjectTypeError && err.data.expected === 'tree' ? 'is not a directory' : 'does not exist'
       throw new Error(`the start path '${startPath}' ${m}`)
-    })
-    .then((result) => Object.assign(result, { dirname: startPath }))
+    }
+  )
 }
 
 function srcGitTree (repo, root, start) {
@@ -505,9 +539,11 @@ function createGitTreeWalker (repo, root, filter) {
     walk (start) {
       return (
         visitGitTree(this, repo, root, filter, start)
-          .then(() => this.emit('end'))
           // NOTE if error is thrown, promises already being resolved won't halt
-          .catch((err) => this.emit('error', err))
+          .then(
+            () => this.emit('end'),
+            (err) => this.emit('error', err)
+          )
       )
     },
   })
@@ -530,8 +566,15 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
         let mode
         if (entry.mode === SYMLINK_FILE_MODE) {
           reads.push(
-            readGitSymlink(repo, root, parent, entry, following)
-              .catch((err) => {
+            readGitSymlink(repo, root, parent, entry, following).then(
+              (target) => {
+                if (target.type === 'tree') {
+                  return visitGitTree(emitter, repo, root, filter, target, vfilePath, new Set(following).add(entry.oid))
+                } else if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
+                  emitter.emit('entry', Object.assign({ mode, oid: target.oid, path: vfilePath }, repo))
+                }
+              },
+              (err) => {
                 // NOTE this error could be caught after promise chain has already been rejected
                 if (err instanceof NotFoundError) {
                   err.message = `Broken symbolic link detected at ${vfilePath}`
@@ -539,14 +582,8 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
                   err.message = `Symbolic link cycle detected at ${vfilePath}`
                 }
                 throw err
-              })
-              .then((target) => {
-                if (target.type === 'tree') {
-                  return visitGitTree(emitter, repo, root, filter, target, vfilePath, new Set(following).add(entry.oid))
-                } else if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
-                  emitter.emit('entry', Object.assign({ mode, oid: target.oid, path: vfilePath }, repo))
-                }
-              })
+              }
+            )
           )
         } else if ((mode = FILE_MODES[entry.mode])) {
           emitter.emit('entry', Object.assign({ mode, oid: entry.oid, path: vfilePath }, repo))
@@ -560,11 +597,11 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
 function readGitSymlink (repo, root, parent, { oid }, following) {
   if (following.size !== (following = new Set(following).add(oid)).size) {
     return git.readBlob(Object.assign({ oid }, repo)).then(({ blob: target }) => {
-      target = posixify && process.env.NODE_ENV === 'test' ? posixify(decodeUint8Data(target)) : decodeUint8Data(target)
+      target = decodeUint8Array(target)
       let targetParent
       if (parent.dirname) {
         const dirname = parent.dirname + '/'
-        target = path.join(dirname, target)
+        target = path.join(dirname, target) // join doesn't remove trailing separator
         if (target.startsWith(dirname)) {
           target = target.substr(dirname.length)
           targetParent = parent
@@ -572,10 +609,12 @@ function readGitSymlink (repo, root, parent, { oid }, following) {
           targetParent = root
         }
       } else {
-        target = path.normalize(target)
+        target = path.normalize(target) // normalize doesn't remove trailing separator
         targetParent = root
       }
-      return readGitObjectAtPath(repo, root, targetParent, target.split('/'), following)
+      const targetSegments = target.split('/')
+      if (!targetSegments[targetSegments.length - 1]) targetSegments.pop()
+      return readGitObjectAtPath(repo, root, targetParent, targetSegments, following)
     })
   }
   const err = { name: 'SymbolicLinkCycleError', code: 'SymbolicLinkCycleError', oid }
@@ -616,11 +655,8 @@ function filterGitEntry (entry) {
 
 function entryToFile (entry) {
   return git.readBlob(entry).then(({ blob: contents }) => {
-    const stat = new fs.Stats()
-    stat.mode = entry.mode
-    stat.mtime = undefined
-    stat.size = contents.byteLength
     contents = Buffer.from(contents.buffer)
+    const stat = Object.assign(new fs.Stats(), { mode: entry.mode, mtime: undefined, size: contents.byteLength })
     return new File({ path: entry.path, contents, stat })
   })
 }
@@ -673,22 +709,23 @@ function loadComponentDescriptor (files, ref, version) {
   return camelCaseKeys(data, { deep: true, stopPaths: ['asciidoc'] })
 }
 
-function computeOrigin (url, authStatus, ref, startPath, worktreePath = undefined, editUrl = true) {
+function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath = undefined, editUrl = true) {
   const { shortname: refname, oid: refhash, type: reftype } = ref
-  const remote = !url.startsWith('file://')
-  const origin = { type: 'git', refname, startPath }
+  const origin = { type: 'git', url, gitdir, refname, [reftype]: refname, startPath }
   if (authStatus) origin.private = authStatus
-  origin[reftype] = refname
-  if (worktreePath) {
-    if (remote) origin.url = url
-    origin.fileUriPattern =
-      (posixify ? 'file:///' + posixify(worktreePath) : 'file://' + worktreePath) + path.join('/', startPath, '%s')
-    origin.worktree = worktreePath
-  } else {
-    origin.url = url
+  if (worktreePath === undefined) {
     origin.refhash = refhash
+  } else {
+    if (worktreePath) {
+      origin.fileUriPattern =
+        (posixify ? 'file:///' + posixify(worktreePath) : 'file://' + worktreePath) + path.join('/', startPath, '%s')
+    } else {
+      origin.refhash = refhash
+    }
+    origin.worktree = worktreePath
+    if (url.startsWith('file://')) url = undefined
   }
-  if (remote) origin.webUrl = url.replace(GIT_SUFFIX_RX, '')
+  if (url) origin.webUrl = url.replace(GIT_SUFFIX_RX, '')
   if (editUrl === true) {
     let match
     if (url && (match = url.match(HOSTED_GIT_REPO_RX))) {
@@ -856,16 +893,20 @@ function generateCloneFolderName (url) {
  *
  * @param {Repository} repo - The repository on which to operate.
  * @param {String} remoteName - The name of the remote to resolve.
- * @returns {String} The URL of the specified remote, if present.
+ * @returns {String} The URL of the specified remote, if defined, or the file URI to the local repository.
  */
 function resolveRemoteUrl (repo, remoteName) {
   return git.getConfig(Object.assign({ path: 'remote.' + remoteName + '.url' }, repo)).then((url) => {
-    if (!url) return posixify ? 'file:///' + posixify(repo.dir) : 'file://' + repo.dir
-    if (url.startsWith('https://') || url.startsWith('http://')) {
-      return ~url.indexOf('@') ? url.replace(URL_AUTH_CLEANER_RX, '$1') : url
-    } else if (url.startsWith('git@')) {
-      return 'https://' + url.substr(4).replace(':', '/')
+    if (url) {
+      if (url.startsWith('https://') || url.startsWith('http://')) {
+        return ~url.indexOf('@') ? url.replace(URL_AUTH_CLEANER_RX, '$1') : url
+      } else if (url.startsWith('git@')) {
+        return 'https://' + url.substr(4).replace(':', '/')
+      } else if (url.startsWith('ssh://')) {
+        return 'https://' + url.substr(url.indexOf('@') + 1 || 6).replace(URL_PORT_CLEANER_RX, '$1')
+      }
     }
+    return posixify ? 'file:///' + posixify(repo.dir) : 'file://' + repo.dir
   })
 }
 
@@ -876,51 +917,18 @@ function resolveRemoteUrl (repo, remoteName) {
  * @return {Boolean} A flag indicating whether the URL matches a directory on the local filesystem.
  */
 function isDirectory (url) {
-  return fsp
-    .stat(url)
-    .then((stat) => stat.isDirectory())
-    .catch(invariably.false)
-}
-
-/**
- * Removes the specified directory (including all of its contents) or file.
- * Equivalent to fs.promises.rmdir(dir, { recursive: true }) in Node 12.
- */
-function rmdir (dir) {
-  return fsp
-    .readdir(dir, { withFileTypes: true })
-    .then((lst) =>
-      Promise.all(
-        lst.map((it) =>
-          it.isDirectory()
-            ? rmdir(ospath.join(dir, it.name))
-            : fsp.unlink(ospath.join(dir, it.name)).catch((unlinkErr) => {
-              if (unlinkErr.code !== 'ENOENT') throw unlinkErr
-            })
-        )
-      )
-    )
-    .then(() => fsp.rmdir(dir))
-    .catch((err) => {
-      if (err.code === 'ENOENT') return
-      if (err.code === 'ENOTDIR') {
-        return fsp.unlink(dir).catch((unlinkErr) => {
-          if (unlinkErr.code !== 'ENOENT') throw unlinkErr
-        })
-      }
-      throw err
-    })
+  return fsp.stat(url).then((stat) => stat.isDirectory(), invariably.false)
 }
 
-function tagsSpecified (sources, defaultTags) {
-  return ~sources.findIndex((source) => {
-    const tags = source.tags || defaultTags || []
-    return Array.isArray(tags) ? tags.length : true
-  })
+function tagsSpecified (sources) {
+  return sources.some(({ tags }) => tags && (Array.isArray(tags) ? tags.length : true))
 }
 
 function loadGitPlugins (gitConfig, networkConfig, startDir) {
-  const plugins = (git.cores || git.default.cores || new Map()).get(GIT_CORE) || new Map()
+  const plugins = new Map((git.cores || git.default.cores || new Map()).get(GIT_CORE))
+  for (const [name, request] of Object.entries(gitConfig.plugins || {})) {
+    if (request) plugins.set(name, userRequire(request, { dot: startDir, paths: [startDir, __dirname] }))
+  }
   let credentialManager, urlRouter
   if ((credentialManager = plugins.get('credentialManager'))) {
     if (typeof credentialManager.configure === 'function') {
@@ -949,19 +957,18 @@ function ensureCacheDir (preferredCacheDir, startDir) {
   const baseCacheDir =
     preferredCacheDir == null
       ? getCacheDir('antora' + (process.env.NODE_ENV === 'test' ? '-test' : '')) || ospath.resolve('.antora/cache')
-      : expandPath(preferredCacheDir, '~+', startDir)
+      : expandPath(preferredCacheDir, { dot: startDir })
   const cacheDir = ospath.join(baseCacheDir, CONTENT_CACHE_FOLDER)
-  return fsp
-    .mkdir(cacheDir, { recursive: true })
-    .then(() => cacheDir)
-    .catch((err) => {
+  return fsp.mkdir(cacheDir, { recursive: true }).then(
+    () => cacheDir,
+    (err) => {
       throw Object.assign(err, { message: `Failed to create content cache directory: ${cacheDir}; ${err.message}` })
-    })
+    }
+  )
 }
 
 function transformGitCloneError (err, displayUrl) {
-  let wrappedMsg
-  let trimMessage
+  let wrappedMsg, trimMessage
   if (HTTP_ERROR_CODE_RX.test(err.code)) {
     switch (err.data.statusCode) {
       case 401:
@@ -992,6 +999,10 @@ function transformGitCloneError (err, displayUrl) {
   return wrappedErr
 }
 
+function splitRefPatterns (str) {
+  return ~str.indexOf('{') ? str.split(VENTILATED_CSV_RX) : str.split(CSV_RX)
+}
+
 function coerceToString (value) {
   return value == null ? '' : String(value)
 }
@@ -1004,11 +1015,11 @@ function findWorktrees (repo, patterns) {
   if (!patterns.length) return new Map()
   const linkedOnly = patterns[0] === '.' ? !(patterns = patterns.slice(1)) : true
   let worktreesDir
+  const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   return (patterns.length
     ? fsp
       .readdir((worktreesDir = ospath.join(repo.dir, '.git', 'worktrees')))
-      .catch(invariably.emptyArray)
-      .then((worktreeNames) => matcher(worktreeNames, [...patterns]))
+      .then((worktreeNames) => filterRefs(worktreeNames, [...patterns], patternCache), invariably.emptyArray)
       .then((worktreeNames) =>
         worktreeNames.length
           ? Promise.all(

package/lib/constants.js

@@ -2,24 +2,25 @@
 
 module.exports = Object.freeze({
   COMPONENT_DESC_FILENAME: 'antora.yml',
-  CONTENT_CACHE_FOLDER: 'content/2',
-  CONTENT_GLOB: '**/*[!~]',
+  CONTENT_CACHE_FOLDER: 'content',
+  CONTENT_SRC_GLOB: '**/*[!~]',
+  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, removeBOM: false, uniqueBy: (m) => m },
   FILE_MODES: { 100644: 0o100666 & ~process.umask(), 100755: 0o100777 & ~process.umask() },
   GIT_CORE: 'antora',
   GIT_OPERATION_LABEL_LENGTH: 8,
   GIT_PROGRESS_PHASES: ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'],
   PICOMATCH_VERSION_OPTS: {
     bash: true,
-    debug: false,
     dot: true,
     fastpaths: false,
-    lookbehinds: false,
-    noextglob: true,
+    nobracket: true,
     noglobstar: true,
     nonegate: true,
     noquantifiers: true,
+    regex: false,
     strictSlashes: true,
   },
+  REF_PATTERN_CACHE_KEY: Symbol('RefPatternCache'),
   SYMLINK_FILE_MODE: '120000',
   VALID_STATE_FILENAME: 'valid',
 })

package/lib/filter-refs.js

@@ -0,0 +1,60 @@
+'use strict'
+
+const { compile: bracesToGroup } = require('braces')
+const { makeRe: makePicomatchRx } = require('picomatch')
+
+function getPicomatchOpts (cache) {
+  return {
+    bash: true,
+    dot: true,
+    expandRange: (begin, end, step, opts) => {
+      const pattern = opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`
+      return cache.braces.get(pattern) || cache.braces.set(pattern, bracesToGroup(pattern)).get(pattern)
+    },
+    fastpaths: false,
+    nobracket: true,
+    noglobstar: true,
+    noquantifiers: true,
+    regex: false,
+    strictSlashes: true,
+  }
+}
+
+function compileRx (pattern, opts) {
+  if (pattern === '*' || pattern === '**') return { test: () => true }
+  return pattern.charAt() === '!' // do our own negate
+    ? Object.defineProperty(makePicomatchRx(pattern.substr(1), opts), 'negated', { value: true })
+    : makePicomatchRx(pattern, opts)
+}
+
+function createMatcher (patterns, cache) {
+  let opts
+  const rxs = patterns.map(
+    (pattern) =>
+      cache.get(pattern) ||
+      cache.set(pattern, compileRx(pattern, opts || (opts = getPicomatchOpts(cache)))).get(pattern)
+  )
+  return (candidate) => {
+    let first = true
+    let matched
+    for (const rx of rxs) {
+      if (matched) {
+        if (rx.negated && rx.test(candidate)) return
+      } else if (first || !rx.negated) {
+        matched = rx.test(candidate)
+      }
+      first = false
+    }
+    return matched
+  }
+}
+
+function filterRefs (candidates, patterns, cache = Object.assign(new Map(), { braces: new Map() })) {
+  const isMatch = createMatcher(patterns, cache)
+  return candidates.reduce((accum, candidate) => {
+    if (isMatch(candidate)) accum.push(candidate)
+    return accum
+  }, [])
+}
+
+module.exports = filterRefs

package/lib/git-credential-manager-store.js

@@ -2,6 +2,7 @@
 
 const { homedir } = require('os')
 const expandPath = require('@antora/expand-path-helper')
+const invariably = { void: () => undefined }
 const { promises: fsp } = require('fs')
 const ospath = require('path')
 
@@ -9,12 +10,10 @@ class GitCredentialManagerStore {
   configure ({ config, startDir }) {
     this.entries = undefined
     this.urls = {}
-    if ((this.contents = (config = config || {}).contents)) {
+    if ((this.contents = (config = config || {}).contents) || !config.path) {
       this.path = undefined
-    } else if (config.path) {
-      this.path = expandPath(config.path, '~+', startDir)
     } else {
-      this.path = undefined
+      this.path = expandPath(config.path, { dot: startDir })
     }
     return this
   }
@@ -28,10 +27,7 @@ class GitCredentialManagerStore {
         contentsPromise = Promise.resolve(this.contents)
         delimiter = /[,\n]/
       } else if (this.path) {
-        contentsPromise = fsp
-          .access(this.path)
-          .then(() => fsp.readFile(this.path, 'utf8'))
-          .catch(() => undefined)
+        contentsPromise = fsp.access(this.path).then(() => fsp.readFile(this.path, 'utf8'), invariably.void)
       } else {
         const homeGitCredentialsPath = ospath.join(homedir(), '.git-credentials')
         const xdgConfigGitCredentialsPath = ospath.join(
@@ -45,8 +41,7 @@ class GitCredentialManagerStore {
           .catch(() =>
             fsp
               .access(xdgConfigGitCredentialsPath)
-              .then(() => fsp.readFile(xdgConfigGitCredentialsPath, 'utf8'))
-              .catch(() => undefined)
+              .then(() => fsp.readFile(xdgConfigGitCredentialsPath, 'utf8'), invariably.void)
           )
       }
       contentsPromise.then((contents) => {

package/lib/git-plugin-http.js

@@ -26,7 +26,7 @@ async function mergeBuffers (data) {
 }
 
 module.exports = ({ httpProxy, httpsProxy, noProxy }, userAgent) => {
-  if (httpsProxy || httpProxy) {
+  if ((httpsProxy || httpProxy) && noProxy !== '*') {
     const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent')
     const shouldProxy = require('should-proxy')
     return {
@@ -34,29 +34,23 @@ module.exports = ({ httpProxy, httpsProxy, noProxy }, userAgent) => {
         headers['user-agent'] = userAgent
         body = await mergeBuffers(body)
         const proxy = url.startsWith('https:')
-          ? { ProxyAgent: HttpsProxyAgent, url: httpsProxy }
-          : { ProxyAgent: HttpProxyAgent, url: httpProxy }
-        let agent
-        if (proxy.url && shouldProxy(url, { no_proxy: noProxy })) {
-          // see https://github.com/delvedor/hpagent/issues/18
-          const { protocol, hostname, port, username, password } = new URL(proxy.url)
-          const proxyUrl = { protocol, hostname, port, username: username || null, password: password || null }
-          agent = new proxy.ProxyAgent({ proxy: proxyUrl })
-        }
+          ? { Agent: HttpsProxyAgent, url: httpsProxy }
+          : { Agent: HttpProxyAgent, url: httpProxy }
+        const agent =
+          proxy.url && shouldProxy(url, { no_proxy: noProxy }) ? new proxy.Agent({ proxy: proxy.url }) : undefined
         return new Promise((resolve, reject) =>
           get({ url, method, agent, headers, body }, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
         )
       },
     }
-  } else {
-    return {
-      async request ({ url, method, headers, body }) {
-        headers['user-agent'] = userAgent
-        body = await mergeBuffers(body)
-        return new Promise((resolve, reject) =>
-          get({ url, method, headers, body }, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
-        )
-      },
-    }
+  }
+  return {
+    async request ({ url, method, headers, body }) {
+      headers['user-agent'] = userAgent
+      body = await mergeBuffers(body)
+      return new Promise((resolve, reject) =>
+        get({ url, method, headers, body }, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
+      )
+    },
   }
 }

package/lib/git.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('isomorphic-git')

package/lib/index.js

@@ -5,7 +5,7 @@
  *
  * Responsible for aggregating the content from multiple repositories and
  * references into a raw aggregate of virtual files that can be organized by a
- * subsequent step in the pipeline.
+ * subsequent step in the generator.
  *
  * @namespace content-aggregator
  */

package/lib/resolve-path-globs.js

@@ -1,26 +1,33 @@
 'use strict'
 
-const { expand: expandBraces } = require('braces')
+const { expand: expandBraces, compile: bracesToGroup } = require('braces')
 const flattenDeep = require('./flatten-deep')
 const { promises: fsp } = require('fs')
-const git = require('isomorphic-git')
+const git = require('./git')
 const invariably = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }
 const { makeRe: makePicomatchRx } = require('picomatch')
 
-const RX_ESCAPE_EXCEPT_GLOB = /[.+?^${}()|[\]\\]/g
-const RX_MAGIC_DETECTOR = /[*{]/
-const RX_QUESTION_MARK = /\?/g
-const PICOMATCH_OPTS = { nobracket: true, noextglob: true, noglobstar: true, noquantifiers: true }
-const PICOMATCH_NEGATED_OPTS = { nobracket: true, noextglob: true, noquantifiers: true }
+const NON_GLOB_SPECIAL_CHARS_RX = /[.+?^${}()|[\]\\]/g
+const RX_MAGIC_DETECTOR = /[*{(]/
+const PICOMATCH_OPTS = {
+  bash: true,
+  expandRange: (begin, end, step, opts) => bracesToGroup(opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`),
+  fastpaths: false,
+  nobracket: true,
+  noglobstar: true,
+  nonegate: true,
+  noquantifiers: true,
+  regex: false,
+  strictSlashes: true,
+}
 
 function resolvePathGlobs (base, patterns, listDirents, retrievePath, tree = { path: '' }) {
   return patterns.reduce((paths, pattern) => {
     if (pattern.charAt() === '!') {
       return paths.then((resolvedPaths) => {
         if (resolvedPaths.length) {
-          if (~pattern.indexOf('?')) pattern = pattern.replace(RX_QUESTION_MARK, '\\?')
-          const rx = makePicomatchRx(pattern, PICOMATCH_NEGATED_OPTS)
-          return resolvedPaths.filter(rx.test.bind(rx))
+          const rx = makePicomatchRx(pattern.substr(1), PICOMATCH_OPTS)
+          return resolvedPaths.filter((it) => !rx.test(it))
         } else {
           return resolvedPaths
         }
@@ -38,14 +45,13 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
   let patternSegment = patternSegments[0]
   patternSegments = patternSegments.slice(1)
   if (RX_MAGIC_DETECTOR.test(patternSegment)) {
-    let isMatch
-    let explicit
+    let isMatch, explicit
     if (patternSegment === '*') {
       isMatch = (it) => it.charAt() !== '.'
+    } else if (~patternSegment.indexOf('(')) {
+      isMatch = (isMatch = makePicomatchRx(patternSegment, PICOMATCH_OPTS)).test.bind(isMatch)
     } else if (~patternSegment.indexOf('{')) {
       if (globbed) {
-        if (patternSegment.charAt() === '!') patternSegment = '\\' + patternSegment
-        if (~patternSegment.indexOf('?')) patternSegment = patternSegment.replace(RX_QUESTION_MARK, '\\?')
         isMatch = (isMatch = makePicomatchRx(patternSegment, PICOMATCH_OPTS)).test.bind(isMatch)
       } else if (~patternSegment.indexOf('*')) {
         const [wildPatterns, literals] = expandBraces(patternSegment).reduce(
@@ -90,22 +96,15 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
         })
       } else if ((patternSegment += '/' + patternSegments.join('/')).indexOf('{')) {
         return expandBraces(patternSegment).map((it) => joinPath(path, it))
-      } else {
-        return [joinPath(path, patternSegment)]
       }
+      return [joinPath(path, patternSegment)]
     } else if (globbed) {
       return (await retrievePath(base, { oid, path }, patternSegment)) ? [joinPath(path, patternSegment)] : []
-    } else {
-      return [joinPath(path, patternSegment)]
     }
+    return [joinPath(path, patternSegment)]
   }
 }
 
-function regexpEscapeWithGlob (str) {
-  // we don't escape "-" since it's meaningless in a literal
-  return str.replace(RX_ESCAPE_EXCEPT_GLOB, '\\$&').replace('*', '.*')
-}
-
 function extractMagicBase (patternSegments, base) {
   let nextSegment
   if (patternSegments.length) {
@@ -124,10 +123,11 @@ function listDirentsFs (base, path) {
 function listDirentsGit (repo, treeOid) {
   return git
     .readTree(Object.assign({ oid: treeOid, filepath: '' }, repo))
-    .then(({ tree: entries }) =>
-      entries.map(({ type, oid, path: name }) => ({ name, oid, isDirectory: invariably[type === 'tree'] }))
+    .then(
+      ({ tree: entries }) =>
+        entries.map(({ type, oid, path: name }) => ({ name, oid, isDirectory: invariably[type === 'tree'] })),
+      invariably.emptyArray
     )
-    .catch(invariably.emptyArray)
 }
 
 function makeAlternationMatcherRx (patterns) {
@@ -139,7 +139,13 @@ function makeMatcherRx (pattern) {
 }
 
 function patternToRx (pattern) {
-  return (pattern.charAt() === '.' ? '' : '(?!\\.)') + regexpEscapeWithGlob(pattern)
+  return (
+    (pattern.charAt() === '.' ? '' : '(?!\\.)') +
+    pattern
+      .replace(NON_GLOB_SPECIAL_CHARS_RX, '\\$&')
+      .replace('\\\\*', '\\x2a')
+      .replace('*', '.*?')
+  )
 }
 
 function readdirWithFileTypes (dir) {
@@ -147,10 +153,7 @@ function readdirWithFileTypes (dir) {
 }
 
 function retrievePathFs (base, { path }, subpath) {
-  return fsp
-    .access(base + '/' + joinPath(path, subpath))
-    .then(invariably.true)
-    .catch(invariably.false)
+  return fsp.access(base + '/' + joinPath(path, subpath)).then(invariably.true, invariably.false)
 }
 
 function retrievePathGit (repo, { oid }, filepath) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.0.0-alpha.7",
+  "version": "3.0.0-beta.2",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -17,14 +17,14 @@
   },
   "main": "lib/index.js",
   "dependencies": {
-    "@antora/expand-path-helper": "~1.0",
+    "@antora/expand-path-helper": "~2.0",
+    "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~6.2",
+    "camelcase-keys": "~7.0",
     "hpagent": "~0.1.0",
-    "isomorphic-git": "~1.8",
+    "isomorphic-git": "~1.10",
     "js-yaml": "~4.1",
-    "matcher": "~4.0",
     "multi-progress": "~4.0",
     "picomatch": "~2.3",
     "progress": "~2.0",
@@ -34,7 +34,7 @@
     "vinyl-fs": "~3.0"
   },
   "engines": {
-    "node": ">=10.17.0"
+    "node": ">=12.21.0"
   },
   "files": [
     "lib/"
@@ -49,5 +49,5 @@
     "static site",
     "web publishing"
   ],
-  "gitHead": "fbd597b3680474f2083cda8a7facf1e2848c08e0"
+  "gitHead": "5cd3f9cc70622e465cb44daf1aa2035ed5a35f54"
 }