@antora/content-aggregator 3.0.3 → 3.1.1

package/lib/aggregate-content.js

@@ -1,14 +1,15 @@
 'use strict'
 
-const camelCaseKeys = require('camelcase-keys')
+const computeOrigin = require('./compute-origin')
 const { createHash } = require('crypto')
 const createGitHttpPlugin = require('./git-plugin-http')
 const decodeUint8Array = require('./decode-uint8-array')
+const deepClone = require('./deep-clone')
+const deepFlatten = require('./deep-flatten')
 const EventEmitter = require('events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
 const filterRefs = require('./filter-refs')
-const flattenDeep = require('./flatten-deep')
 const fs = require('fs')
 const { promises: fsp } = fs
 const getCacheDir = require('cache-directory')
@@ -17,11 +18,13 @@ const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
 const globStream = require('glob-stream')
 const invariably = require('./invariably')
+const logger = require('./logger')
 const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')
 const MultiProgress = require('multi-progress') // calls require('progress') as a peer dependencies
 const ospath = require('path')
 const { posix: path } = ospath
-const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
+const posixify = require('./posixify')
+const removeGitSuffix = require('./remove-git-suffix')
 const { fs: resolvePathGlobsFs, git: resolvePathGlobsGit } = require('./resolve-path-globs')
 const { pipeline, Writable } = require('stream')
 const forEach = (write) => new Writable({ objectMode: true, write })
@@ -44,12 +47,9 @@ const {
 const ANY_SEPARATOR_RX = /[:/]/
 const CSV_RX = /\s*,\s*/
 const VENTILATED_CSV_RX = /\s*,\s+/
-const EDIT_URL_TEMPLATE_VAR_RX = /\{(web_url|ref(?:hash|name)|path)\}/g
-const GIT_SUFFIX_RX = /(?:(?:(?:\.git)?\/)?\.git|\/)$/
 const GIT_URI_DETECTOR_RX = /:(?:\/\/|[^/\\])/
-const HEADS_DIR_RX = /^heads\//
-const HOSTED_GIT_REPO_RX = /^(?:https?:\/\/|.+@)(git(?:hub|lab)\.com|bitbucket\.org|pagure\.io)[/:](.+?)(?:\.git)?$/
 const HTTP_ERROR_CODE_RX = new RegExp('^' + git.Errors.HttpError.code + '$', 'i')
+const NEWLINE_RX = /\n/g
 const PATH_SEPARATOR_RX = /[/]/g
 const SHORTEN_REF_RX = /^refs\/(?:heads|remotes\/[^/]+|tags)\//
 const SPACE_RX = / /g
@@ -154,13 +154,14 @@ async function collectFiles (sourcesByUrl, loadOpts, concurrency) {
 
 function buildAggregate (componentVersionBuckets) {
   return [
-    ...flattenDeep(componentVersionBuckets)
+    ...deepFlatten(componentVersionBuckets)
       .reduce((accum, batch) => {
         const key = batch.version + '@' + batch.name
         const entry = accum.get(key)
         if (!entry) return accum.set(key, batch)
-        const files = batch.files
+        const { files, origins } = batch
         ;(batch.files = entry.files).push(...files)
+        ;(batch.origins = entry.origins).push(origins[0])
         Object.assign(entry, batch)
         return accum
       }, new Map())
@@ -257,9 +258,17 @@ function extractCredentials (url) {
 
 async function collectFilesFromSource (source, repo, remoteName, authStatus) {
   const originUrl = repo.url || (await resolveRemoteUrl(repo, remoteName))
-  return selectReferences(source, repo, remoteName).then((refs) =>
-    Promise.all(refs.map((ref) => collectFilesFromReference(source, repo, remoteName, authStatus, ref, originUrl)))
-  )
+  return selectReferences(source, repo, remoteName).then((refs) => {
+    if (!refs.length) {
+      const { url, branches, tags, startPath, startPaths } = source
+      const startPathInfo =
+        'startPaths' in source ? { 'start paths': startPaths || undefined } : { 'start path': startPath || undefined }
+      const sourceInfo = yaml.dump({ url, branches, tags, ...startPathInfo }, { flowLevel: 1 }).trimRight()
+      logger.info(`No matching references found for content source entry (${sourceInfo.replace(NEWLINE_RX, ' | ')})`)
+      return []
+    }
+    return Promise.all(refs.map((it) => collectFilesFromReference(source, repo, remoteName, authStatus, it, originUrl)))
+  })
 }
 
 // QUESTION should we resolve HEAD to a ref eagerly to avoid having to do a match on it?
@@ -267,7 +276,7 @@ async function selectReferences (source, repo, remote) {
   let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns = '.' } = source
   const isBare = repo.noCheckout
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
-  const noWorktree = repo.url ? undefined : null
+  const noWorktree = repo.url ? undefined : false
   const refs = new Map()
   if (
     tagPatterns &&
@@ -403,8 +412,9 @@ async function collectFilesFromReference (source, repo, remoteName, authStatus,
       ? resolvePathGlobsFs(worktreePath, startPaths)
       : resolvePathGlobsGit(repo, ref.oid, startPaths))
     if (!startPaths.length) {
-      const refInfo = `ref: ${ref.fullname.replace(HEADS_DIR_RX, '')}${worktreePath ? ' <worktree>' : ''}`
-      throw new Error(`no start paths found in ${displayUrl} (${refInfo})`)
+      const where = worktreePath || (worktreePath === false ? repo.gitdir : displayUrl)
+      const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
+      throw new Error(`no start paths found in ${where} (${ref.type}: ${ref.shortname}${flag})`)
     }
     return Promise.all(
       startPaths.map((startPath) =>
@@ -417,29 +427,30 @@ async function collectFilesFromReference (source, repo, remoteName, authStatus,
 }
 
 function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version) {
-  return (
-    worktreePath ? readFilesFromWorktree(worktreePath, startPath) : readFilesFromGitTree(repo, ref.oid, startPath)
-  )
-    .then((files) => {
-      const componentVersionBucket = loadComponentDescriptor(files, ref, version)
-      const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
-      componentVersionBucket.files = files.map((file) => assignFileProperties(file, origin))
-      return componentVersionBucket
-    })
+  const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
+  return (worktreePath ? readFilesFromWorktree(origin) : readFilesFromGitTree(repo, ref.oid, startPath))
+    .then((files) =>
+      Object.assign(deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version))), {
+        files: files.map((file) => assignFileProperties(file, origin)),
+        origins: [origin],
+      })
+    )
     .catch((err) => {
-      const msg = err.message
-      const refInfo = `ref: ${ref.fullname.replace(HEADS_DIR_RX, '')}${worktreePath ? ' <worktree>' : ''}`
-      const pathInfo = !startPath || msg.startsWith('the start path ') ? '' : ' | path: ' + startPath
-      throw Object.assign(err, { message: msg.replace(/$/m, ` in ${repo.url || repo.dir} (${refInfo}${pathInfo})`) })
+      const where = worktreePath || (worktreePath === false ? repo.gitdir : repo.url || repo.dir)
+      const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
+      const pathInfo = startPath ? (err.message.startsWith('the start path ') ? '' : ` | start path: ${startPath}`) : ''
+      const message = err.message.replace(/$/m, ` in ${where} (${ref.type}: ${ref.shortname}${flag}${pathInfo})`)
+      throw Object.assign(err, { message })
     })
 }
 
-function readFilesFromWorktree (worktreePath, startPath) {
-  const cwd = ospath.join(worktreePath, startPath, '.') // . shaves off trailing slash
+function readFilesFromWorktree (origin) {
+  const startPath = origin.startPath
+  const cwd = ospath.join(origin.worktree, startPath, '.') // . shaves off trailing slash
   return fsp.stat(cwd).then(
     (startPathStat) => {
       if (!startPathStat.isDirectory()) throw new Error(`the start path '${startPath}' is not a directory`)
-      return srcFs(cwd)
+      return srcFs(cwd, origin)
     },
     () => {
       throw new Error(`the start path '${startPath}' does not exist`)
@@ -447,38 +458,45 @@ function readFilesFromWorktree (worktreePath, startPath) {
   )
 }
 
-function srcFs (cwd) {
-  const relpathStart = cwd.length + 1
-  return new Promise((resolve, reject, cache = Object.create(null), files = []) =>
+function srcFs (cwd, origin) {
+  return new Promise((resolve, reject, cache = Object.create(null), files = [], relpathStart = cwd.length + 1) =>
     pipeline(
       globStream(CONTENT_SRC_GLOB, Object.assign({ cache, cwd }, CONTENT_SRC_OPTS)),
       forEach(({ path: abspathPosix }, _, done) => {
-        if (Array.isArray(cache[abspathPosix])) return done() // detects some directories, but not all
+        if ((cache[abspathPosix] || {}).constructor === Array) return done() // detects some directories
         const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
         const relpath = abspath.substr(relpathStart)
         symlinkAwareStat(abspath).then(
           (stat) => {
-            if (stat.isDirectory()) return done() // detects remaining directories
+            if (stat.isDirectory()) return done() // detects directories that slipped through cache check
             fsp.readFile(abspath).then(
               (contents) => {
                 files.push(new File({ path: posixify ? posixify(relpath) : relpath, contents, stat, src: { abspath } }))
                 done()
               },
               (readErr) => {
-                done(Object.assign(readErr, { message: readErr.message.replace(`'${abspath}'`, relpath) }))
+                const logObject = { file: { abspath, origin } }
+                readErr.code === 'ENOENT'
+                  ? logger.warn(logObject, `ENOENT: file or directory disappeared, ${readErr.syscall} ${relpath}`)
+                  : logger.error(logObject, readErr.message.replace(`'${abspath}'`, relpath))
+                done()
               }
             )
           },
           (statErr) => {
+            const logObject = { file: { abspath, origin } }
             if (statErr.symlink) {
-              statErr.message =
-                statErr.code === 'ELOOP'
-                  ? `Symbolic link cycle detected at ${relpath}`
-                  : `Broken symbolic link detected at ${relpath}`
+              logger.error(
+                logObject,
+                (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
+                  `${relpath} -> ${statErr.symlink}`
+              )
+            } else if (statErr.code === 'ENOENT') {
+              logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
             } else {
-              statErr.message = statErr.message.replace(`'${abspath}'`, relpath)
+              logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
             }
-            done(statErr)
+            done()
           }
         )
       }),
@@ -510,30 +528,26 @@ function getGitTreeAtStartPath (repo, oid, startPath) {
 function srcGitTree (repo, root, start) {
   return new Promise((resolve, reject) => {
     const files = []
-    createGitTreeWalker(repo, root, filterGitEntry)
-      .on('entry', (entry) => files.push(entryToFile(entry)))
+    createGitTreeWalker(repo, root, filterGitEntry, gitEntryToFile)
+      .on('entry', (file) => files.push(file))
       .on('error', reject)
-      .on('end', () => resolve(Promise.all(files)))
+      .on('end', () => resolve(files))
       .walk(start)
   })
 }
 
-function createGitTreeWalker (repo, root, filter) {
+function createGitTreeWalker (repo, root, filter, convert) {
   return Object.assign(new EventEmitter(), {
     walk (start) {
-      return (
-        visitGitTree(this, repo, root, filter, start)
-          // NOTE if error is thrown, promises already being resolved won't halt
-          .then(
-            () => this.emit('end'),
-            (err) => this.emit('error', err)
-          )
+      return visitGitTree(this, repo, root, filter, convert, start).then(
+        () => this.emit('end'),
+        (err) => this.emit('error', err)
       )
     },
   })
 }
 
-function visitGitTree (emitter, repo, root, filter, parent, dirname = '', following = new Set()) {
+function visitGitTree (emitter, repo, root, filter, convert, parent, dirname = '', following = new Set()) {
   const reads = []
   for (const entry of parent.tree) {
     const filterVerdict = filter(entry)
@@ -543,7 +557,7 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
         reads.push(
           git.readTree(Object.assign({ oid: entry.oid }, repo)).then((subtree) => {
             Object.assign(subtree, { dirname: path.join(parent.dirname, entry.path) })
-            return visitGitTree(emitter, repo, root, filter, subtree, vfilePath, following)
+            return visitGitTree(emitter, repo, root, filter, convert, subtree, vfilePath, following)
           })
         )
       } else if (entry.type === 'blob') {
@@ -553,56 +567,70 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
             readGitSymlink(repo, root, parent, entry, following).then(
               (target) => {
                 if (target.type === 'tree') {
-                  return visitGitTree(emitter, repo, root, filter, target, vfilePath, new Set(following).add(entry.oid))
+                  return visitGitTree(emitter, repo, root, filter, convert, target, vfilePath, target.following)
                 } else if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
-                  emitter.emit('entry', Object.assign({ mode, oid: target.oid, path: vfilePath }, repo))
+                  return convert(Object.assign({ mode, oid: target.oid, path: vfilePath }, repo)).then((result) =>
+                    emitter.emit('entry', result)
+                  )
                 }
               },
               (err) => {
-                // NOTE this error could be caught after promise chain has already been rejected
-                if (err instanceof NotFoundError) {
-                  err.message = `Broken symbolic link detected at ${vfilePath}`
-                } else if (err.code === 'SymbolicLinkCycleError') {
-                  err.message = `Symbolic link cycle detected at ${vfilePath}`
+                if (err.symlink) {
+                  err.message =
+                    (err.code === 'ELOOP' ? 'ELOOP: symbolic link cycle' : 'ENOENT: broken symbolic link') +
+                    `, ${vfilePath} -> ${err.symlink}`
                 }
                 throw err
               }
             )
           )
         } else if ((mode = FILE_MODES[entry.mode])) {
-          emitter.emit('entry', Object.assign({ mode, oid: entry.oid, path: vfilePath }, repo))
+          reads.push(
+            convert(Object.assign({ mode, oid: entry.oid, path: vfilePath }, repo)).then((result) =>
+              emitter.emit('entry', result)
+            )
+          )
         }
       }
     }
   }
-  return Promise.all(reads)
+  // NOTE preserve scan order so error for symbolic link cycle is deterministic; ensures no rejections after resolve
+  return Promise.allSettled(reads).then((results) => {
+    const rejected = results.find(({ reason }) => reason)
+    if (rejected) throw rejected.reason
+  })
 }
 
-function readGitSymlink (repo, root, parent, { oid }, following) {
-  if (following.size !== (following = new Set(following).add(oid)).size) {
-    return git.readBlob(Object.assign({ oid }, repo)).then(({ blob: target }) => {
-      target = decodeUint8Array(target)
-      let targetParent
-      if (parent.dirname) {
-        const dirname = parent.dirname + '/'
-        target = path.join(dirname, target) // join doesn't remove trailing separator
-        if (target.startsWith(dirname)) {
-          target = target.substr(dirname.length)
-          targetParent = parent
-        } else {
-          targetParent = root
-        }
-      } else {
-        target = path.normalize(target) // normalize doesn't remove trailing separator
-        targetParent = root
+function readGitSymlink (repo, root, parent, { oid, path: name }, following) {
+  const dirname = parent.dirname
+  if (following.size === (following = new Set(following)).add(oid).size) {
+    const err = { name: 'SymbolicLinkCycleError', code: 'ELOOP', oid, path: `${path.join(dirname, name)}` }
+    return Promise.reject(Object.assign(new Error(`Symbolic link cycle detected at ${oid}:${err.path}`), err))
+  }
+  return git.readBlob(Object.assign({ oid }, repo)).then(({ blob: symlink }) => {
+    symlink = decodeUint8Array(symlink)
+    let target
+    let targetParent = root
+    if (dirname) {
+      if (!(target = path.join('/', dirname, symlink).substr(1)) || target === dirname) {
+        target = '.'
+      } else if (target.startsWith(dirname + '/')) {
+        target = target.substr(dirname.length + 1) // join doesn't remove trailing separator
+        targetParent = parent
       }
-      const targetSegments = target.split('/')
-      if (!targetSegments[targetSegments.length - 1]) targetSegments.pop()
-      return readGitObjectAtPath(repo, root, targetParent, targetSegments, following)
+    } else {
+      target = path.normalize(symlink) // normalize doesn't remove trailing separator
+    }
+    if (target === '.') {
+      const err = { name: 'SymbolicLinkCycleError', code: 'ELOOP', oid, path: `${path.join(dirname, name)}`, symlink }
+      return Promise.reject(Object.assign(new Error(`Symbolic link cycle detected at ${oid}:${err.path}`), err))
+    }
+    const targetSegments = target.split('/')
+    targetSegments[targetSegments.length - 1] || targetSegments.pop()
+    return readGitObjectAtPath(repo, root, targetParent, targetSegments, following).catch((err) => {
+      throw Object.assign(err, { symlink })
     })
-  }
-  const err = { name: 'SymbolicLinkCycleError', code: 'SymbolicLinkCycleError', oid }
-  return Promise.reject(Object.assign(new Error(`Symbolic link cycle found at oid: ${err.oid}`), err))
+  })
 }
 
 // QUESTION: could we use this to resolve the start path too?
@@ -615,14 +643,14 @@ function readGitObjectAtPath (repo, root, parent, pathSegments, following) {
           Object.assign(subtree, { dirname: path.join(parent.dirname, entry.path) })
           return (pathSegments = pathSegments.slice(1)).length
             ? readGitObjectAtPath(repo, root, subtree, pathSegments, following)
-            : Object.assign(subtree, { type: 'tree' })
+            : Object.assign(subtree, { type: 'tree', following }) // Q: should this create copy?
         })
         : entry.mode === SYMLINK_FILE_MODE
           ? readGitSymlink(repo, root, parent, entry, following)
           : Promise.resolve(entry)
     }
   }
-  return Promise.reject(new NotFoundError(`No file or directory found at "${parent.oid}:${pathSegments.join('/')}"`))
+  return Promise.reject(new NotFoundError(`${parent.oid}:${pathSegments.join('/')}`))
 }
 
 /**
@@ -637,7 +665,7 @@ function filterGitEntry (entry) {
   return entryPath.charAt(entryPath.length - 1) !== '~'
 }
 
-function entryToFile (entry) {
+function gitEntryToFile (entry) {
   return git.readBlob(entry).then(({ blob: contents }) => {
     contents = Buffer.from(contents.buffer)
     const stat = Object.assign(new fs.Stats(), { mode: entry.mode, mtime: undefined, size: contents.byteLength })
@@ -691,53 +719,7 @@ function loadComponentDescriptor (files, ref, version) {
     throw new Error(`version in ${COMPONENT_DESC_FILENAME} cannot have path segments: ${version}`)
   }
   data.version = version
-  return camelCaseKeys(data, { deep: true, stopPaths: ['asciidoc'] })
-}
-
-function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath = undefined, editUrl = true) {
-  const { shortname: refname, oid: refhash, type: reftype } = ref
-  const origin = { type: 'git', url, gitdir, refname, [reftype]: refname, startPath }
-  if (authStatus) origin.private = authStatus
-  if (worktreePath === undefined) {
-    origin.refhash = refhash
-  } else {
-    if (worktreePath) {
-      origin.fileUriPattern =
-        (posixify ? 'file:///' + posixify(worktreePath) : 'file://' + worktreePath) +
-        (startPath ? '/' + startPath + '/%s' : '/%s')
-    } else {
-      origin.refhash = refhash
-    }
-    origin.worktree = worktreePath
-    if (url.startsWith('file://')) url = undefined
-  }
-  if (url) origin.webUrl = url.replace(GIT_SUFFIX_RX, '')
-  if (editUrl === true) {
-    let match
-    if (url && (match = url.match(HOSTED_GIT_REPO_RX))) {
-      const host = match[1]
-      let action
-      let category = ''
-      if (host === 'pagure.io') {
-        action = 'blob'
-        category = 'f'
-      } else if (host === 'bitbucket.org') {
-        action = 'src'
-      } else {
-        action = reftype === 'branch' ? 'edit' : 'blob'
-      }
-      origin.editUrlPattern = 'https://' + path.join(match[1], match[2], action, refname, category, startPath, '%s')
-    }
-  } else if (editUrl) {
-    const vars = {
-      path: () => (startPath ? path.join(startPath, '%s') : '%s'),
-      refhash: () => refhash,
-      refname: () => refname,
-      web_url: () => origin.webUrl || '',
-    }
-    origin.editUrlPattern = editUrl.replace(EDIT_URL_TEMPLATE_VAR_RX, (_, name) => vars[name]())
-  }
-  return origin
+  return camelCaseKeys(data, ['asciidoc'])
 }
 
 function assignFileProperties (file, origin) {
@@ -823,17 +805,17 @@ function onGitProgress ({ phase, loaded, total }) {
     const scaleFactor = this.scaleFactor
     let ratio = ((loaded / total) * scaleFactor) / GIT_PROGRESS_PHASES.length
     if (phaseIdx) ratio += (phaseIdx * scaleFactor) / GIT_PROGRESS_PHASES.length
-    // NOTE: updates are automatically throttled based on renderThrottle option
+    // NOTE updates are automatically throttled based on renderThrottle option
     this.update(ratio > scaleFactor ? scaleFactor : ratio)
   }
 }
 
 function onGitComplete (err) {
   if (err) {
-    // TODO: could use progressBar.interrupt() to replace bar with message instead
+    // TODO could use progressBar.interrupt() to replace bar with message instead
     this.chars.incomplete = '?'
     this.update(0)
-    // NOTE: force progress bar to update regardless of throttle setting
+    // NOTE force progress bar to update regardless of throttle setting
     this.render(undefined, true)
   } else {
     this.update(1)
@@ -869,13 +851,9 @@ function resolveCredentials (credentialsFromUrlHolder, url, auth) {
  * @returns {String} The generated folder name.
  */
 function generateCloneFolderName (url) {
-  let normalizedUrl = url.toLowerCase()
-  if (posixify) normalizedUrl = posixify(normalizedUrl)
-  normalizedUrl = normalizedUrl.replace(GIT_SUFFIX_RX, '')
+  const normalizedUrl = removeGitSuffix(posixify ? posixify(url.toLowerCase()) : url.toLowerCase())
   const basename = normalizedUrl.split(ANY_SEPARATOR_RX).pop()
-  const hash = createHash('sha1')
-  hash.update(normalizedUrl)
-  return basename + '-' + hash.digest('hex') + '.git'
+  return basename + '-' + createHash('sha1').update(normalizedUrl).digest('hex') + '.git'
 }
 
 /**
@@ -914,9 +892,14 @@ function isDirectory (url) {
 function symlinkAwareStat (path_) {
   return fsp.lstat(path_).then((lstat) => {
     if (!lstat.isSymbolicLink()) return lstat
-    return fsp.stat(path_).catch((statErr) => {
-      throw Object.assign(statErr, { symlink: true })
-    })
+    return fsp.stat(path_).catch((statErr) =>
+      fsp
+        .readlink(path_)
+        .catch(invariably.void)
+        .then((symlink) => {
+          throw Object.assign(statErr, { symlink })
+        })
+    )
   })
 }
 
@@ -994,23 +977,35 @@ function transformGitCloneError (err, displayUrl) {
   if (trimMessage) {
     wrappedMsg = ~(wrappedMsg = wrappedMsg.trimRight()).indexOf('. ') ? wrappedMsg : wrappedMsg.replace(/\.$/, '')
   }
-  const wrappedErr = new Error(`${wrappedMsg} (url: ${displayUrl})`)
-  wrappedErr.stack += `\nCaused by: ${err.stack || 'unknown'}`
-  return wrappedErr
+  const errWrapper = new Error(`${wrappedMsg} (url: ${displayUrl})`)
+  errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
+  return errWrapper
 }
 
 function splitRefPatterns (str) {
   return ~str.indexOf('{') ? str.split(VENTILATED_CSV_RX) : str.split(CSV_RX)
 }
 
-function coerceToString (value) {
-  return value == null ? '' : String(value)
+function camelCaseKeys (o, stopPaths = [], p = '') {
+  if (Array.isArray(o)) return o.map((it) => camelCaseKeys(it, stopPaths, p))
+  if (o == null || o.constructor !== Object) return o
+  const pathPrefix = p && p + '.'
+  const accum = {}
+  for (const [k, v] of Object.entries(o)) {
+    const camelKey = k.toLowerCase().replace(/[_-]([a-z0-9])/g, (_, l, idx) => (idx ? l.toUpperCase() : l))
+    accum[camelKey] = ~stopPaths.indexOf(pathPrefix + camelKey) ? v : camelCaseKeys(v, stopPaths, pathPrefix + camelKey)
+  }
+  return accum
 }
 
 function cleanStartPath (value) {
   return value && ~value.indexOf('/') ? value.replace(SUPERFLUOUS_SEPARATORS_RX, '') : value
 }
 
+function coerceToString (value) {
+  return value == null ? '' : String(value)
+}
+
 function findWorktrees (repo, patterns) {
   if (!patterns.length) return new Map()
   const linkedOnly = patterns[0] === '.' ? !(patterns = patterns.slice(1)) : true
@@ -1047,4 +1042,3 @@ function findWorktrees (repo, patterns) {
 }
 
 module.exports = aggregateContent
-module.exports._computeOrigin = computeOrigin

package/lib/compute-origin.js

@@ -0,0 +1,54 @@
+'use strict'
+
+const { posix: path } = require('path')
+const posixify = require('./posixify')
+const removeGitSuffix = require('./remove-git-suffix')
+
+const EDIT_URL_TEMPLATE_VAR_RX = /\{(web_url|ref(?:hash|name|type)|path)\}/g
+const HOSTED_GIT_REPO_RX = /^(?:https?:\/\/|.+@)(git(?:hub|lab)\.com|bitbucket\.org|pagure\.io)[/:](.+?)(?:\.git)?$/
+
+function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath = undefined, editUrl = true) {
+  const { shortname: refname, oid: refhash, remote, type: reftype } = ref
+  const origin = { type: 'git', url, gitdir, reftype, refname, [reftype]: refname, refhash, startPath }
+  if (worktreePath !== undefined) {
+    if ((origin.worktree = worktreePath)) {
+      delete origin.refhash
+      origin.fileUriPattern =
+        (posixify ? 'file:///' + posixify(worktreePath) : 'file://' + worktreePath) +
+        (startPath ? '/' + startPath + '/%s' : '/%s')
+    } else if (remote) {
+      origin.remote = remote
+    }
+    if (url.startsWith('file://')) url = undefined
+  }
+  if (authStatus) origin.private = authStatus
+  if (url) origin.webUrl = removeGitSuffix(url)
+  if (editUrl === true) {
+    const match = url && url.match(HOSTED_GIT_REPO_RX)
+    if (match) {
+      const host = match[1]
+      let action = 'blob'
+      let category = ''
+      if (host === 'pagure.io') {
+        category = 'f'
+      } else if (host === 'bitbucket.org') {
+        action = 'src'
+      } else if (reftype === 'branch') {
+        action = 'edit'
+      }
+      origin.editUrlPattern = 'https://' + path.join(match[1], match[2], action, refname, category, startPath, '%s')
+    }
+  } else if (editUrl) {
+    const vars = {
+      path: () => (startPath ? path.join(startPath, '%s') : '%s'),
+      refhash: () => refhash,
+      reftype: () => reftype,
+      refname: () => refname,
+      web_url: () => origin.webUrl || '',
+    }
+    origin.editUrlPattern = editUrl.replace(EDIT_URL_TEMPLATE_VAR_RX, (_, name) => vars[name]())
+  }
+  return origin
+}
+
+module.exports = computeOrigin

package/lib/constants.js

@@ -4,7 +4,7 @@ module.exports = Object.freeze({
   COMPONENT_DESC_FILENAME: 'antora.yml',
   CONTENT_CACHE_FOLDER: 'content',
   CONTENT_SRC_GLOB: '**/*[!~]',
-  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, strict: false, uniqueBy: (m) => m },
+  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, strict: false },
   FILE_MODES: { 100644: 0o100666 & ~process.umask(), 100755: 0o100777 & ~process.umask() },
   GIT_CORE: 'antora',
   GIT_OPERATION_LABEL_LENGTH: 8,

package/lib/deep-clone.js

@@ -0,0 +1,18 @@
+'use strict'
+
+function deepClone (o) {
+  switch (o.constructor) {
+    case Object:
+      return Object.keys(o).reduce((accum, k) => {
+        const v = o[k]
+        accum[k] = !v || typeof v !== 'object' ? v : deepClone(v)
+        return accum
+      }, {})
+    case Array:
+      return o.map((it) => (!it || typeof it !== 'object' ? it : deepClone(it)))
+    default:
+      return o
+  }
+}
+
+module.exports = deepClone

package/lib/{flatten-deep.js → deep-flatten.js}

@@ -1,9 +1,9 @@
 'use strict'
 
-function flattenDeep (array, accum = []) {
+function deepFlatten (array, accum = []) {
   const len = array.length
-  for (let i = 0, it; i < len; i++) Array.isArray((it = array[i])) ? flattenDeep(it, accum) : accum.push(it)
+  for (let i = 0, it; i < len; i++) Array.isArray((it = array[i])) ? deepFlatten(it, accum) : accum.push(it)
   return accum
 }
 
-module.exports = flattenDeep
+module.exports = deepFlatten

package/lib/logger.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('@antora/logger')(require('../package.json').name)

package/lib/posixify.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('path').sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined

package/lib/remove-git-suffix.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const GIT_SUFFIX_RX = /(?:(?:(?:\.git)?\/)?\.git|\/)$/
+
+function removeGitSuffix (url) {
+  return url.replace(GIT_SUFFIX_RX, '')
+}
+
+module.exports = removeGitSuffix

package/lib/resolve-path-globs.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const flattenDeep = require('./flatten-deep')
+const deepFlatten = require('./deep-flatten')
 const { promises: fsp } = require('fs')
 const git = require('./git')
 const invariably = require('./invariably')
@@ -56,7 +56,7 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
     }
     let dirents = await listDirents(base, oid || path)
     if (explicit) dirents = dirents.filter((dirent) => !explicit.has(dirent.name))
-    const discovered = flattenDeep(
+    const discovered = deepFlatten(
       await Promise.all(
         dirents.map((dirent) =>
           dirent.isDirectory() && isMatch(dirent.name)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.0.3",
+  "version": "3.1.1",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -16,15 +16,26 @@
     "url": "https://gitlab.com/antora/antora/issues"
   },
   "main": "lib/index.js",
+  "exports": {
+    ".": "./lib/index.js",
+    "./git": "./lib/git.js",
+    "./git/http-plugin": "./lib/git-plugin-http.js",
+    "./lib/git-plugin-http": "./lib/git-plugin-http.js",
+    "./package.json": "./package.json"
+  },
+  "imports": {
+    "#compute-origin": "./lib/compute-origin.js",
+    "#constants": "./lib/constants.js"
+  },
   "dependencies": {
     "@antora/expand-path-helper": "~2.0",
+    "@antora/logger": "3.1.1",
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~7.0",
     "glob-stream": "~7.0",
-    "hpagent": "~0.1.0",
-    "isomorphic-git": "~1.10",
+    "hpagent": "~1.0",
+    "isomorphic-git": "~1.19",
     "js-yaml": "~4.1",
     "multi-progress": "~4.0",
     "picomatch": "~2.3",
@@ -34,7 +45,7 @@
     "vinyl": "~2.2"
   },
   "engines": {
-    "node": ">=12.21.0"
+    "node": ">=16.0.0"
   },
   "files": [
     "lib/"