@antora/content-aggregator 3.1.0 → 3.2.0-alpha.1

package/lib/aggregate-content.js

@@ -188,8 +188,7 @@ async function loadRepository (url, opts) {
         await git
           .fetch(fetchOpts)
           .then(() => {
-            const credentialManager = gitPlugins.credentialManager
-            authStatus = credentials ? 'auth-embedded' : credentialManager.status({ url }) ? 'auth-required' : undefined
+            authStatus = identifyAuthStatus(gitPlugins.credentialManager, credentials, url)
             return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
           })
           .catch((fetchErr) => {
@@ -210,8 +209,7 @@ async function loadRepository (url, opts) {
         .clone(fetchOpts)
         .then(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo)))
         .then(() => {
-          const credentialManager = gitPlugins.credentialManager
-          authStatus = credentials ? 'auth-embedded' : credentialManager.status({ url }) ? 'auth-required' : undefined
+          authStatus = identifyAuthStatus(gitPlugins.credentialManager, credentials, url)
           return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
         })
         .catch((cloneErr) => {
@@ -223,8 +221,15 @@ async function loadRepository (url, opts) {
         .then(() => fetchOpts.onProgress && fetchOpts.onProgress.finish())
     }
   } else if (await isDirectory((dir = expandPath(url, { dot: opts.startDir })))) {
-    const gitdir = ospath.join(dir, '.git')
-    repo = (await isDirectory(gitdir)) ? { cache, dir, fs, gitdir } : { cache, dir, fs, gitdir: dir, noCheckout: true }
+    const dotgit = ospath.join(dir, '.git')
+    const dotgitStat = await fsp.stat(dotgit).catch(() => ({ isFile: invariably.false, isDirectory: invariably.false }))
+    if (dotgitStat.isDirectory()) {
+      repo = { cache, dir, fs, gitdir: dotgit }
+    } else if (dotgitStat.isFile()) {
+      repo = await resolveRepositoryFromWorktree({ cache, dir, fs, gitdir: dotgit })
+    } else {
+      repo = { cache, dir, fs, gitdir: dir, noCheckout: true }
+    }
     try {
       await git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo))
     } catch {
@@ -273,7 +278,7 @@ async function collectFilesFromSource (source, repo, remoteName, authStatus) {
 
 // QUESTION should we resolve HEAD to a ref eagerly to avoid having to do a match on it?
 async function selectReferences (source, repo, remote) {
-  let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns = '.' } = source
+  let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns } = source
   const isBare = repo.noCheckout
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   const noWorktree = repo.url ? undefined : false
@@ -293,6 +298,12 @@ async function selectReferences (source, repo, remote) {
     }
   }
   if (!branchPatterns) return [...refs.values()]
+  branchPatterns = Array.isArray(branchPatterns)
+    ? branchPatterns.map((pattern) => String(pattern))
+    : splitRefPatterns(String(branchPatterns))
+  if (!branchPatterns.length) return [...refs.values()]
+  const worktreeName = repo.worktreeName // possibly switch to worktree property ({ name, dir}) in future
+  if (worktreeName) branchPatterns = branchPatterns.map((it) => (it === 'HEAD' ? 'HEAD@' + worktreeName : it))
   if (worktreePatterns) {
     if (worktreePatterns === '.') {
       worktreePatterns = ['.']
@@ -302,10 +313,12 @@ async function selectReferences (source, repo, remote) {
       worktreePatterns = Array.isArray(worktreePatterns)
         ? worktreePatterns.map((pattern) => String(pattern))
         : splitRefPatterns(String(worktreePatterns))
+      if (worktreeName) worktreePatterns = worktreePatterns.map((it) => (it === '@' ? worktreeName : it))
     }
+  } else {
+    worktreePatterns = worktreePatterns === undefined ? [worktreeName || '.'] : []
   }
-  const branchPatternsString = String(branchPatterns)
-  if (branchPatternsString === 'HEAD' || branchPatternsString === '.') {
+  if (branchPatterns.length === 1 && (branchPatterns[0] === 'HEAD' || branchPatterns[0] === '.')) {
     const currentBranch = await getCurrentBranchName(repo, remote)
     if (currentBranch) {
       branchPatterns = [currentBranch]
@@ -317,11 +330,7 @@ async function selectReferences (source, repo, remote) {
       refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
       return [...refs.values()]
     }
-  } else if (
-    (branchPatterns = Array.isArray(branchPatterns)
-      ? branchPatterns.map((pattern) => String(pattern))
-      : splitRefPatterns(branchPatternsString)).length
-  ) {
+  } else {
     let headBranchIdx
     // NOTE we can assume at least two entries if HEAD or . are present
     if (~(headBranchIdx = branchPatterns.indexOf('HEAD')) || ~(headBranchIdx = branchPatterns.indexOf('.'))) {
@@ -346,8 +355,6 @@ async function selectReferences (source, repo, remote) {
         branchPatterns.splice(headBranchIdx, 1)
       }
     }
-  } else {
-    return [...refs.values()]
   }
   // NOTE isomorphic-git includes HEAD in list of remote branches (see https://isomorphic-git.org/docs/listBranches)
   const remoteBranches = (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
@@ -362,8 +369,20 @@ async function selectReferences (source, repo, remote) {
     const localBranches = await git.listBranches(repo)
     if (localBranches.length) {
       const worktrees = await findWorktrees(repo, worktreePatterns)
-      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
-        const head = worktrees.get(shortname) || noWorktree
+      let onMatch
+      if ((worktreePatterns.join('') || '.') !== '.') {
+        const symbolicNames = new Map()
+        worktrees.forEach(({ name, symbolicName = 'HEAD@' + name }, shortname) => {
+          localBranches.push(symbolicName)
+          symbolicNames.set(symbolicName, shortname)
+        })
+        onMatch = (candidate, { pattern }) => {
+          const shortname = symbolicNames.get(candidate)
+          return shortname ? (pattern.startsWith('HEAD@') ? shortname : undefined) : candidate
+        }
+      }
+      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache, onMatch)) {
+        const head = (worktrees.get(shortname) || { head: noWorktree }).head
         refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
       }
     }
@@ -382,15 +401,13 @@ async function selectReferences (source, repo, remote) {
  * Returns the current branch name unless the HEAD is detached.
  */
 function getCurrentBranchName (repo, remote) {
-  let refPromise
-  if (repo.noCheckout) {
-    refPromise = git
-      .resolveRef(Object.assign({ ref: 'refs/remotes/' + remote + '/HEAD', depth: 2 }, repo))
-      .catch(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 2 }, repo)))
-  } else {
-    refPromise = git.resolveRef(Object.assign({ ref: 'HEAD', depth: 2 }, repo))
-  }
-  return refPromise.then((ref) => (ref.startsWith('refs/') ? ref.replace(SHORTEN_REF_RX, '') : undefined))
+  return (
+    repo.noCheckout && remote
+      ? git
+        .resolveRef(Object.assign({ ref: 'refs/remotes/' + remote + '/HEAD', depth: 2 }, repo))
+        .catch(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 2 }, repo)))
+      : git.resolveRef(Object.assign({ ref: 'HEAD', depth: 2 }, repo))
+  ).then((ref) => (ref.startsWith('refs/') ? ref.replace(SHORTEN_REF_RX, '') : undefined))
 }
 
 async function collectFilesFromReference (source, repo, remoteName, authStatus, ref, originUrl) {
@@ -459,17 +476,16 @@ function readFilesFromWorktree (origin) {
 }
 
 function srcFs (cwd, origin) {
-  const relpathStart = cwd.length + 1
-  return new Promise((resolve, reject, cache = Object.create(null), files = []) =>
+  return new Promise((resolve, reject, cache = Object.create(null), files = [], relpathStart = cwd.length + 1) =>
     pipeline(
       globStream(CONTENT_SRC_GLOB, Object.assign({ cache, cwd }, CONTENT_SRC_OPTS)),
       forEach(({ path: abspathPosix }, _, done) => {
-        if (Array.isArray(cache[abspathPosix])) return done() // detects some directories, but not all
+        if ((cache[abspathPosix] || {}).constructor === Array) return done() // detects some directories
         const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
         const relpath = abspath.substr(relpathStart)
         symlinkAwareStat(abspath).then(
           (stat) => {
-            if (stat.isDirectory()) return done() // detects remaining directories
+            if (stat.isDirectory()) return done() // detects directories that slipped through cache check
             fsp.readFile(abspath).then(
               (contents) => {
                 files.push(new File({ path: posixify ? posixify(relpath) : relpath, contents, stat, src: { abspath } }))
@@ -824,11 +840,14 @@ function onGitComplete (err) {
 }
 
 function resolveCredentials (credentialsFromUrlHolder, url, auth) {
-  const credentialsFromUrl = credentialsFromUrlHolder.get()
+  const credentialsFromUrl = credentialsFromUrlHolder.get() || {}
   if ('Authorization' in auth.headers) {
-    if (!credentialsFromUrl) return this.rejected({ url, auth })
-    credentialsFromUrlHolder.clear()
-  } else if (credentialsFromUrl) {
+    if ('username' in credentialsFromUrl) {
+      credentialsFromUrlHolder.clear()
+    } else {
+      return this.rejected({ url, auth })
+    }
+  } else if ('username' in credentialsFromUrl) {
     return credentialsFromUrl
   } else {
     auth = undefined
@@ -840,6 +859,15 @@ function resolveCredentials (credentialsFromUrlHolder, url, auth) {
   )
 }
 
+function identifyAuthStatus (credentialManager, credentials, url) {
+  const status = credentialManager.status({ url })
+  if (credentials) {
+    return typeof status === 'string' && status.startsWith('requested,') ? 'auth-required' : 'auth-embedded'
+  } else if (status != null) {
+    return 'auth-required'
+  }
+}
+
 /**
  * Generates a safe, unique folder name for a git URL.
  *
@@ -852,13 +880,9 @@ function resolveCredentials (credentialsFromUrlHolder, url, auth) {
  * @returns {String} The generated folder name.
  */
 function generateCloneFolderName (url) {
-  let normalizedUrl = url.toLowerCase()
-  if (posixify) normalizedUrl = posixify(normalizedUrl)
-  normalizedUrl = removeGitSuffix(normalizedUrl)
+  const normalizedUrl = removeGitSuffix(posixify ? posixify(url.toLowerCase()) : url.toLowerCase())
   const basename = normalizedUrl.split(ANY_SEPARATOR_RX).pop()
-  const hash = createHash('sha1')
-  hash.update(normalizedUrl)
-  return basename + '-' + hash.digest('hex') + '.git'
+  return basename + '-' + createHash('sha1').update(normalizedUrl).digest('hex') + '.git'
 }
 
 /**
@@ -888,7 +912,7 @@ function resolveRemoteUrl (repo, remoteName) {
  * Checks whether the specified URL matches a directory on the local filesystem.
  *
  * @param {String} url - The URL to check.
- * @return {Boolean} A flag indicating whether the URL matches a directory on the local filesystem.
+ * @returns {Boolean} A flag indicating whether the URL matches a directory on the local filesystem.
  */
 function isDirectory (url) {
   return fsp.stat(url).then((stat) => stat.isDirectory(), invariably.false)
@@ -1011,39 +1035,54 @@ function coerceToString (value) {
   return value == null ? '' : String(value)
 }
 
+async function resolveRepositoryFromWorktree (repo) {
+  return fsp
+    .readFile(repo.gitdir, 'utf8')
+    .then((contents) => contents.trimRight().substr(8))
+    .then((worktreeGitdir) => {
+      const worktreeName = ospath.basename(worktreeGitdir)
+      return fsp.readFile(ospath.join(worktreeGitdir, 'commondir'), 'utf8').then(
+        (contents) => {
+          const gitdir = ospath.join(worktreeGitdir, contents.trimRight())
+          return ospath.basename(gitdir) === '.git'
+            ? Object.assign(repo, { dir: ospath.dirname(gitdir), gitdir, worktreeName })
+            : Object.assign(repo, { dir: gitdir, gitdir, worktreeName })
+        },
+        () => repo
+      )
+    })
+}
+
 function findWorktrees (repo, patterns) {
   if (!patterns.length) return new Map()
-  const linkedOnly = patterns[0] === '.' ? !(patterns = patterns.slice(1)) : true
-  let worktreesDir
+  const mainWorktree =
+    patterns[0] === '.' && (patterns = patterns.slice(1))
+      ? getCurrentBranchName(repo).then((branch) => (branch ? [branch, { head: repo.dir, name: '.' }] : undefined))
+      : Promise.resolve()
+  const worktreesDir = patterns.length ? ospath.join(repo.dir, '.git', 'worktrees') : undefined
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
   return (
-    patterns.length
+    worktreesDir
       ? fsp
-        .readdir((worktreesDir = ospath.join(repo.dir, '.git', 'worktrees')))
+        .readdir(worktreesDir)
         .then((worktreeNames) => filterRefs(worktreeNames, patterns, patternCache), invariably.emptyArray)
         .then((worktreeNames) =>
-          worktreeNames.length
-            ? Promise.all(
-              worktreeNames.map((worktreeName) => {
-                const gitdir = ospath.resolve(worktreesDir, worktreeName)
-                // NOTE uses name of worktree as branch name if HEAD is detached
-                return git
-                  .currentBranch(Object.assign({}, repo, { gitdir }))
-                  .then((branch = worktreeName) =>
-                    fsp
-                      .readFile(ospath.join(gitdir, 'gitdir'), 'utf8')
-                      .then((contents) => ({ branch, dir: ospath.dirname(contents.trimRight()) }))
-                  )
-              })
-            ).then((entries) => entries.reduce((accum, it) => accum.set(it.branch, it.dir), new Map()))
-            : new Map()
+          Promise.all(
+            worktreeNames.map((worktreeName) => {
+              const gitdir = ospath.resolve(worktreesDir, worktreeName)
+              // NOTE branch name defaults to worktree name if HEAD is detached
+              return git
+                .currentBranch(Object.assign({}, repo, { gitdir }))
+                .then((branch = worktreeName) =>
+                  fsp
+                    .readFile(ospath.join(gitdir, 'gitdir'), 'utf8')
+                    .then((contents) => [branch, { head: ospath.dirname(contents.trimRight()), name: worktreeName }])
+                )
+            })
+          )
         )
-      : Promise.resolve(new Map())
-  ).then((worktrees) =>
-    linkedOnly
-      ? worktrees
-      : git.currentBranch(repo).then((branch) => (branch ? worktrees.set(branch, repo.dir) : worktrees))
-  )
+      : Promise.resolve()
+  ).then((entries = []) => mainWorktree.then((entry) => new Map(entry ? entries.push(entry) && entries : entries)))
 }
 
 module.exports = aggregateContent

package/lib/compute-origin.js

@@ -4,7 +4,7 @@ const { posix: path } = require('path')
 const posixify = require('./posixify')
 const removeGitSuffix = require('./remove-git-suffix')
 
-const EDIT_URL_TEMPLATE_VAR_RX = /\{(web_url|ref(?:hash|name|type)|path)\}/g
+const EDIT_URL_TEMPLATE_VAR_RX = /\{(web_url|ref(?:hash|name|type)?|path)\}/g
 const HOSTED_GIT_REPO_RX = /^(?:https?:\/\/|.+@)(git(?:hub|lab)\.com|bitbucket\.org|pagure\.io)[/:](.+?)(?:\.git)?$/
 
 function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath = undefined, editUrl = true) {
@@ -41,6 +41,7 @@ function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath =
   } else if (editUrl) {
     const vars = {
       path: () => (startPath ? path.join(startPath, '%s') : '%s'),
+      ref: () => 'refs/' + (reftype === 'branch' ? 'heads' : reftype) + '/' + refname,
       refhash: () => refhash,
       reftype: () => reftype,
       refname: () => refname,

package/lib/constants.js

@@ -4,7 +4,7 @@ module.exports = Object.freeze({
   COMPONENT_DESC_FILENAME: 'antora.yml',
   CONTENT_CACHE_FOLDER: 'content',
   CONTENT_SRC_GLOB: '**/*[!~]',
-  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, strict: false, uniqueBy: (m) => m },
+  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, strict: false },
   FILE_MODES: { 100644: 0o100666 & ~process.umask(), 100755: 0o100777 & ~process.umask() },
   GIT_CORE: 'antora',
   GIT_OPERATION_LABEL_LENGTH: 8,

package/lib/filter-refs.js

@@ -4,9 +4,11 @@ const { makeMatcherRx, refMatcherOpts: getMatcherOpts, MATCH_ALL_RX } = require(
 
 function compileRx (pattern, opts) {
   if (pattern === '*' || pattern === '**') return MATCH_ALL_RX
-  return pattern.charAt() === '!' // do our own negate
-    ? Object.defineProperty(makeMatcherRx(pattern.substr(1), opts), 'negated', { value: true })
-    : makeMatcherRx(pattern, opts)
+  const rx =
+    pattern.charAt() === '!' // we handle negate ourselves
+      ? Object.defineProperty(makeMatcherRx((pattern = pattern.substr(1)), opts), 'negated', { value: true })
+      : makeMatcherRx(pattern, opts)
+  return Object.defineProperty(rx, 'pattern', { value: pattern })
 }
 
 function createMatcher (patterns, cache, opts) {
@@ -15,8 +17,8 @@ function createMatcher (patterns, cache, opts) {
       cache.get(pattern) || cache.set(pattern, compileRx(pattern, opts || (opts = getMatcherOpts(cache)))).get(pattern)
   )
   if (rxs[0].negated) rxs.unshift(MATCH_ALL_RX)
-  return (candidate) => {
-    let matched
+  return (candidate, onMatch) => {
+    let matched, symbolic
     for (const rx of rxs) {
       let voteIfMatched = true
       if (matched) {
@@ -25,16 +27,22 @@ function createMatcher (patterns, cache, opts) {
       } else if (rx.negated) {
         continue
       }
-      if (rx.test(candidate)) matched = voteIfMatched
+      if (rx.test(candidate) || (symbolic && rx.test(symbolic) && (candidate = symbolic))) {
+        if (onMatch) {
+          if (!(matched = onMatch(candidate, rx))) continue
+          ;[symbolic, candidate] = [candidate, matched]
+        }
+        matched = voteIfMatched && candidate
+      }
     }
     return matched
   }
 }
 
-function filterRefs (candidates, patterns, cache = Object.assign(new Map(), { braces: new Map() })) {
-  const isMatch = createMatcher(patterns, cache)
+function filterRefs (candidates, patterns, cache = Object.assign(new Map(), { braces: new Map() }), onMatch) {
+  const match = createMatcher(patterns, cache)
   return candidates.reduce((accum, candidate) => {
-    if (isMatch(candidate)) accum.push(candidate)
+    if ((candidate = match(candidate, onMatch))) accum.push(candidate)
     return accum
   }, [])
 }

package/lib/git-credential-manager-store.js

@@ -86,11 +86,11 @@ class GitCredentialManagerStore {
   }
 
   async approved ({ url }) {
-    this.urls[url] = 'approved'
+    this.urls[url] = (url in this.urls ? this.urls[url] + ',' : '') + 'approved'
   }
 
   async rejected ({ url, auth }) {
-    this.urls[url] = 'rejected'
+    this.urls[url] = (url in this.urls ? this.urls[url] + ',' : '') + 'rejected'
     const statusCode = 401
     const statusMessage = 'HTTP Basic: Access Denied'
     const err = new Error(`HTTP Error: ${statusCode} ${statusMessage}`)

package/lib/matcher.js

@@ -22,7 +22,7 @@ function makeMatcherRx (input, opts) {
 }
 
 module.exports = {
-  MATCH_ALL_RX: { test: () => true },
+  MATCH_ALL_RX: Object.defineProperty({ test: () => true }, 'pattern', { value: '*' }),
   expandBraces,
   makeMatcherRx,
   pathMatcherOpts: Object.assign({}, BASE_OPTS, { dot: false }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.1.0",
+  "version": "3.2.0-alpha.1",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -29,13 +29,13 @@
   },
   "dependencies": {
     "@antora/expand-path-helper": "~2.0",
-    "@antora/logger": "3.1.0",
+    "@antora/logger": "3.2.0-alpha.1",
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
     "glob-stream": "~7.0",
-    "hpagent": "~1.0",
-    "isomorphic-git": "~1.19",
+    "hpagent": "~1.1",
+    "isomorphic-git": "~1.21",
     "js-yaml": "~4.1",
     "multi-progress": "~4.0",
     "picomatch": "~2.3",