@antora/content-aggregator 3.0.0-alpha.9 → 3.0.0-beta.1

package/README.md

@@ -3,7 +3,7 @@
 The Content Aggregator is a component in Antora responsible for fetching and aggregating content distributed across multiple local and remote git repositories for use in an Antora documentation pipeline.
 
 [Antora](https://antora.org) is a modular static site generator designed for creating documentation sites from AsciiDoc documents.
-Its site generator pipeline aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
+Its site generator aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
 
 ## Copyright and License
 

package/lib/aggregate-content.js

@@ -9,6 +9,7 @@ const decodeUint8Array = require('./decode-uint8-array')
 const EventEmitter = require('events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
+const filterRefs = require('./filter-refs')
 const flattenDeep = require('./flatten-deep')
 const fs = require('fs')
 const { promises: fsp } = fs
@@ -18,7 +19,6 @@ const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
 const invariably = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }
 const { makeRe: makePicomatchRx } = require('picomatch')
-const matcher = require('matcher')
 const MultiProgress = require('multi-progress')
 const ospath = require('path')
 const { posix: path } = ospath
@@ -40,6 +40,7 @@ const {
   GIT_OPERATION_LABEL_LENGTH,
   GIT_PROGRESS_PHASES,
   PICOMATCH_VERSION_OPTS,
+  REF_PATTERN_CACHE_KEY,
   SYMLINK_FILE_MODE,
   VALID_STATE_FILENAME,
 } = require('./constants')
@@ -101,7 +102,8 @@ function aggregateContent (playbook) {
       return accum.set(source.url, [...(accum.get(source.url) || []), Object.assign({}, sourceDefaults, source)])
     }, new Map())
     const progress = !quiet && createProgress(sourcesByUrl.keys(), process.stdout)
-    const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir }
+    const refPatternCache = Object.assign(new Map(), { braces: new Map() })
+    const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir, refPatternCache }
     return collectFiles(sourcesByUrl, loadOpts, fetchConcurrency).then(buildAggregate, (err) => {
       progress && progress.terminate()
       throw err
@@ -173,13 +175,14 @@ function buildAggregate (componentVersionBuckets) {
 
 async function loadRepository (url, opts) {
   let authStatus, dir, repo
+  const cache = { [REF_PATTERN_CACHE_KEY]: opts.refPatternCache }
   if (~url.indexOf(':') && GIT_URI_DETECTOR_RX.test(url)) {
     let credentials, displayUrl
     ;({ displayUrl, url, credentials } = extractCredentials(url))
     const { cacheDir, fetch, fetchTags, gitPlugins, progress } = opts
     dir = ospath.join(cacheDir, generateCloneFolderName(displayUrl))
     // NOTE the presence of the url property on the repo object implies the repository is remote
-    repo = { cache: {}, dir, fs, gitdir: dir, noCheckout: true, url }
+    repo = { cache, dir, fs, gitdir: dir, noCheckout: true, url }
     const validStateFile = ospath.join(dir, VALID_STATE_FILENAME)
     try {
       await fsp.access(validStateFile)
@@ -204,7 +207,7 @@ async function loadRepository (url, opts) {
         authStatus = await git.getConfig(Object.assign({ path: 'remote.origin.private' }, repo))
       }
     } catch (gitErr) {
-      await rmdir(dir)
+      await fsp['rm' in fsp ? 'rm' : 'rmdir'](dir, { recursive: true, force: true })
       if (gitErr.rethrow) throw transformGitCloneError(gitErr, displayUrl)
       const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetchTags, 'clone')
       await git
@@ -225,9 +228,7 @@ async function loadRepository (url, opts) {
     }
   } else if (await isDirectory((dir = expandPath(url, { dot: opts.startDir })))) {
     const gitdir = ospath.join(dir, '.git')
-    repo = (await isDirectory(gitdir))
-      ? { cache: {}, dir, fs, gitdir }
-      : { cache: {}, dir, fs, gitdir: dir, noCheckout: true }
+    repo = (await isDirectory(gitdir)) ? { cache, dir, fs, gitdir } : { cache, dir, fs, gitdir: dir, noCheckout: true }
     try {
       await git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo))
     } catch {
@@ -271,15 +272,16 @@ async function collectFilesFromSource (source, repo, remoteName, authStatus) {
 async function selectReferences (source, repo, remote) {
   let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns = '.' } = source
   const isBare = repo.noCheckout
+  const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   const noWorktree = repo.url ? undefined : null
   const refs = new Map()
   if (tagPatterns) {
     tagPatterns = Array.isArray(tagPatterns)
       ? tagPatterns.map((pattern) => String(pattern))
-      : String(tagPatterns).split(CSV_RX)
+      : splitRefPatterns(String(tagPatterns))
     if (tagPatterns.length) {
       const tags = await git.listTags(repo)
-      for (const shortname of tags.length ? matcher(tags, tagPatterns) : tags) {
+      for (const shortname of tags.length ? filterRefs(tags, tagPatterns, patternCache) : tags) {
         // NOTE tags are stored using symbol keys to distinguish them from branches
         refs.set(Symbol(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
       }
@@ -294,7 +296,7 @@ async function selectReferences (source, repo, remote) {
       } else {
         worktreePatterns = Array.isArray(worktreePatterns)
           ? worktreePatterns.map((pattern) => String(pattern))
-          : String(worktreePatterns).split(CSV_RX)
+          : splitRefPatterns(String(worktreePatterns))
       }
     }
     const branchPatternsString = String(branchPatterns)
@@ -313,7 +315,7 @@ async function selectReferences (source, repo, remote) {
     } else if (
       (branchPatterns = Array.isArray(branchPatterns)
         ? branchPatterns.map((pattern) => String(pattern))
-        : branchPatternsString.split(CSV_RX)).length
+        : splitRefPatterns(branchPatternsString)).length
     ) {
       let headBranchIdx
       // NOTE we can assume at least two entries if HEAD or . are present
@@ -345,7 +347,7 @@ async function selectReferences (source, repo, remote) {
     // NOTE isomorphic-git includes HEAD in list of remote branches (see https://isomorphic-git.org/docs/listBranches)
     const remoteBranches = (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
     if (remoteBranches.length) {
-      for (const shortname of matcher(remoteBranches, branchPatterns)) {
+      for (const shortname of filterRefs(remoteBranches, branchPatterns, patternCache)) {
         const fullname = 'remotes/' + remote + '/' + shortname
         refs.set(shortname, { shortname, fullname, type: 'branch', remote, head: noWorktree })
       }
@@ -355,16 +357,17 @@ async function selectReferences (source, repo, remote) {
       const localBranches = await git.listBranches(repo)
       if (localBranches.length) {
         const worktrees = await findWorktrees(repo, worktreePatterns)
-        for (const shortname of matcher(localBranches, branchPatterns)) {
+        for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
           const head = worktrees.get(shortname) || noWorktree
           refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
         }
       }
     } else if (!remoteBranches.length) {
-      // QUESTION should local branches be used if the only remote branch is HEAD?
       const localBranches = await git.listBranches(repo)
-      for (const shortname of localBranches.length ? matcher(localBranches, branchPatterns) : localBranches) {
-        refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head: noWorktree })
+      if (localBranches.length) {
+        for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
+          refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head: noWorktree })
+        }
       }
     }
   }
@@ -601,7 +604,7 @@ function readGitSymlink (repo, root, parent, { oid }, following) {
       let targetParent
       if (parent.dirname) {
         const dirname = parent.dirname + '/'
-        target = path.join(dirname, target)
+        target = path.join(dirname, target) // join doesn't remove trailing separator
         if (target.startsWith(dirname)) {
           target = target.substr(dirname.length)
           targetParent = parent
@@ -609,10 +612,12 @@ function readGitSymlink (repo, root, parent, { oid }, following) {
           targetParent = root
         }
       } else {
-        target = path.normalize(target)
+        target = path.normalize(target) // normalize doesn't remove trailing separator
         targetParent = root
       }
-      return readGitObjectAtPath(repo, root, targetParent, target.split('/'), following)
+      const targetSegments = target.split('/')
+      if (!targetSegments[targetSegments.length - 1]) targetSegments.pop()
+      return readGitObjectAtPath(repo, root, targetParent, targetSegments, following)
     })
   }
   const err = { name: 'SymbolicLinkCycleError', code: 'SymbolicLinkCycleError', oid }
@@ -918,36 +923,6 @@ function isDirectory (url) {
   return fsp.stat(url).then((stat) => stat.isDirectory(), invariably.false)
 }
 
-/**
- * Removes the specified directory (including all of its contents) or file.
- * Equivalent to fs.promises.rmdir(dir, { recursive: true }) in Node 12.
- */
-function rmdir (dir) {
-  return fsp
-    .readdir(dir, { withFileTypes: true })
-    .then((lst) =>
-      Promise.all(
-        lst.map((it) =>
-          it.isDirectory()
-            ? rmdir(ospath.join(dir, it.name))
-            : fsp.unlink(ospath.join(dir, it.name)).catch((unlinkErr) => {
-              if (unlinkErr.code !== 'ENOENT') throw unlinkErr
-            })
-        )
-      )
-    )
-    .then(() => fsp.rmdir(dir))
-    .catch((err) => {
-      if (err.code === 'ENOENT') return
-      if (err.code === 'ENOTDIR') {
-        return fsp.unlink(dir).catch((unlinkErr) => {
-          if (unlinkErr.code !== 'ENOENT') throw unlinkErr
-        })
-      }
-      throw err
-    })
-}
-
 function tagsSpecified (sources) {
   return sources.some(({ tags }) => tags && (Array.isArray(tags) ? tags.length : true))
 }
@@ -1027,6 +1002,10 @@ function transformGitCloneError (err, displayUrl) {
   return wrappedErr
 }
 
+function splitRefPatterns (str) {
+  return ~str.indexOf('{') ? str.split(VENTILATED_CSV_RX) : str.split(CSV_RX)
+}
+
 function coerceToString (value) {
   return value == null ? '' : String(value)
 }
@@ -1039,10 +1018,11 @@ function findWorktrees (repo, patterns) {
   if (!patterns.length) return new Map()
   const linkedOnly = patterns[0] === '.' ? !(patterns = patterns.slice(1)) : true
   let worktreesDir
+  const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   return (patterns.length
     ? fsp
       .readdir((worktreesDir = ospath.join(repo.dir, '.git', 'worktrees')))
-      .then((worktreeNames) => matcher(worktreeNames, [...patterns]), invariably.emptyArray)
+      .then((worktreeNames) => filterRefs(worktreeNames, [...patterns], patternCache), invariably.emptyArray)
       .then((worktreeNames) =>
         worktreeNames.length
           ? Promise.all(

package/lib/constants.js

@@ -11,16 +11,16 @@ module.exports = Object.freeze({
   GIT_PROGRESS_PHASES: ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'],
   PICOMATCH_VERSION_OPTS: {
     bash: true,
-    debug: false,
     dot: true,
     fastpaths: false,
-    lookbehinds: false,
-    noextglob: true,
+    nobracket: true,
     noglobstar: true,
     nonegate: true,
     noquantifiers: true,
+    regex: false,
     strictSlashes: true,
   },
+  REF_PATTERN_CACHE_KEY: Symbol('RefPatternCache'),
   SYMLINK_FILE_MODE: '120000',
   VALID_STATE_FILENAME: 'valid',
 })

package/lib/filter-refs.js

@@ -0,0 +1,60 @@
+'use strict'
+
+const { compile: bracesToGroup } = require('braces')
+const { makeRe: makePicomatchRx } = require('picomatch')
+
+function getPicomatchOpts (cache) {
+  return {
+    bash: true,
+    dot: true,
+    expandRange: (begin, end, step, opts) => {
+      const pattern = opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`
+      return cache.braces.get(pattern) || cache.braces.set(pattern, bracesToGroup(pattern)).get(pattern)
+    },
+    fastpaths: false,
+    nobracket: true,
+    noglobstar: true,
+    noquantifiers: true,
+    regex: false,
+    strictSlashes: true,
+  }
+}
+
+function compileRx (pattern, opts) {
+  if (pattern === '*' || pattern === '**') return { test: () => true }
+  return pattern.charAt() === '!' // do our own negate
+    ? Object.defineProperty(makePicomatchRx(pattern.substr(1), opts), 'negated', { value: true })
+    : makePicomatchRx(pattern, opts)
+}
+
+function createMatcher (patterns, cache) {
+  let opts
+  const rxs = patterns.map(
+    (pattern) =>
+      cache.get(pattern) ||
+      cache.set(pattern, compileRx(pattern, opts || (opts = getPicomatchOpts(cache)))).get(pattern)
+  )
+  return (candidate) => {
+    let first = true
+    let matched
+    for (const rx of rxs) {
+      if (matched) {
+        if (rx.negated && rx.test(candidate)) return
+      } else if (first || !rx.negated) {
+        matched = rx.test(candidate)
+      }
+      first = false
+    }
+    return matched
+  }
+}
+
+function filterRefs (candidates, patterns, cache = Object.assign(new Map(), { braces: new Map() })) {
+  const isMatch = createMatcher(patterns, cache)
+  return candidates.reduce((accum, candidate) => {
+    if (isMatch(candidate)) accum.push(candidate)
+    return accum
+  }, [])
+}
+
+module.exports = filterRefs

package/lib/index.js

@@ -5,7 +5,7 @@
  *
  * Responsible for aggregating the content from multiple repositories and
  * references into a raw aggregate of virtual files that can be organized by a
- * subsequent step in the pipeline.
+ * subsequent step in the generator.
  *
  * @namespace content-aggregator
  */

package/lib/resolve-path-globs.js

@@ -1,26 +1,33 @@
 'use strict'
 
-const { expand: expandBraces } = require('braces')
+const { expand: expandBraces, compile: bracesToGroup } = require('braces')
 const flattenDeep = require('./flatten-deep')
 const { promises: fsp } = require('fs')
 const git = require('./git')
 const invariably = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }
 const { makeRe: makePicomatchRx } = require('picomatch')
 
-const RX_ESCAPE_EXCEPT_GLOB = /[.+?^${}()|[\]\\]/g
-const RX_MAGIC_DETECTOR = /[*{]/
-const RX_QUESTION_MARK = /\?/g
-const PICOMATCH_OPTS = { nobracket: true, noextglob: true, noglobstar: true, noquantifiers: true }
-const PICOMATCH_NEGATED_OPTS = { nobracket: true, noextglob: true, noquantifiers: true }
+const NON_GLOB_SPECIAL_CHARS_RX = /[.+?^${}()|[\]\\]/g
+const RX_MAGIC_DETECTOR = /[*{(]/
+const PICOMATCH_OPTS = {
+  bash: true,
+  expandRange: (begin, end, step, opts) => bracesToGroup(opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`),
+  fastpaths: false,
+  nobracket: true,
+  noglobstar: true,
+  nonegate: true,
+  noquantifiers: true,
+  regex: false,
+  strictSlashes: true,
+}
 
 function resolvePathGlobs (base, patterns, listDirents, retrievePath, tree = { path: '' }) {
   return patterns.reduce((paths, pattern) => {
     if (pattern.charAt() === '!') {
       return paths.then((resolvedPaths) => {
         if (resolvedPaths.length) {
-          if (~pattern.indexOf('?')) pattern = pattern.replace(RX_QUESTION_MARK, '\\?')
-          const rx = makePicomatchRx(pattern, PICOMATCH_NEGATED_OPTS)
-          return resolvedPaths.filter(rx.test.bind(rx))
+          const rx = makePicomatchRx(pattern.substr(1), PICOMATCH_OPTS)
+          return resolvedPaths.filter((it) => !rx.test(it))
         } else {
           return resolvedPaths
         }
@@ -38,14 +45,13 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
   let patternSegment = patternSegments[0]
   patternSegments = patternSegments.slice(1)
   if (RX_MAGIC_DETECTOR.test(patternSegment)) {
-    let isMatch
-    let explicit
+    let isMatch, explicit
     if (patternSegment === '*') {
       isMatch = (it) => it.charAt() !== '.'
+    } else if (~patternSegment.indexOf('(')) {
+      isMatch = (isMatch = makePicomatchRx(patternSegment, PICOMATCH_OPTS)).test.bind(isMatch)
     } else if (~patternSegment.indexOf('{')) {
       if (globbed) {
-        if (patternSegment.charAt() === '!') patternSegment = '\\' + patternSegment
-        if (~patternSegment.indexOf('?')) patternSegment = patternSegment.replace(RX_QUESTION_MARK, '\\?')
         isMatch = (isMatch = makePicomatchRx(patternSegment, PICOMATCH_OPTS)).test.bind(isMatch)
       } else if (~patternSegment.indexOf('*')) {
         const [wildPatterns, literals] = expandBraces(patternSegment).reduce(
@@ -90,22 +96,15 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
         })
       } else if ((patternSegment += '/' + patternSegments.join('/')).indexOf('{')) {
         return expandBraces(patternSegment).map((it) => joinPath(path, it))
-      } else {
-        return [joinPath(path, patternSegment)]
       }
+      return [joinPath(path, patternSegment)]
     } else if (globbed) {
       return (await retrievePath(base, { oid, path }, patternSegment)) ? [joinPath(path, patternSegment)] : []
-    } else {
-      return [joinPath(path, patternSegment)]
     }
+    return [joinPath(path, patternSegment)]
   }
 }
 
-function regexpEscapeWithGlob (str) {
-  // we don't escape "-" since it's meaningless in a literal
-  return str.replace(RX_ESCAPE_EXCEPT_GLOB, '\\$&').replace('*', '.*')
-}
-
 function extractMagicBase (patternSegments, base) {
   let nextSegment
   if (patternSegments.length) {
@@ -140,7 +139,13 @@ function makeMatcherRx (pattern) {
 }
 
 function patternToRx (pattern) {
-  return (pattern.charAt() === '.' ? '' : '(?!\\.)') + regexpEscapeWithGlob(pattern)
+  return (
+    (pattern.charAt() === '.' ? '' : '(?!\\.)') +
+    pattern
+      .replace(NON_GLOB_SPECIAL_CHARS_RX, '\\$&')
+      .replace('\\\\*', '\\x2a')
+      .replace('*', '.*?')
+  )
 }
 
 function readdirWithFileTypes (dir) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.0.0-alpha.9",
+  "version": "3.0.0-beta.1",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -21,11 +21,10 @@
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~6.2",
+    "camelcase-keys": "~7.0",
     "hpagent": "~0.1.0",
     "isomorphic-git": "~1.10",
     "js-yaml": "~4.1",
-    "matcher": "~4.0",
     "multi-progress": "~4.0",
     "picomatch": "~2.3",
     "progress": "~2.0",
@@ -35,7 +34,7 @@
     "vinyl-fs": "~3.0"
   },
   "engines": {
-    "node": ">=10.17.0"
+    "node": ">=12.21.0"
   },
   "files": [
     "lib/"
@@ -50,5 +49,5 @@
     "static site",
     "web publishing"
   ],
-  "gitHead": "a504d6889819b548e8a5416a7194cbb6f9a93e93"
+  "gitHead": "7c5ef1ea93dd489af533c80a936c736013c41769"
 }