@antora/content-aggregator 3.0.1 → 3.1.0

package/lib/aggregate-content.js

@@ -1,14 +1,15 @@
 'use strict'
 
-const camelCaseKeys = require('camelcase-keys')
+const computeOrigin = require('./compute-origin')
 const { createHash } = require('crypto')
 const createGitHttpPlugin = require('./git-plugin-http')
 const decodeUint8Array = require('./decode-uint8-array')
+const deepClone = require('./deep-clone')
+const deepFlatten = require('./deep-flatten')
 const EventEmitter = require('events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
 const filterRefs = require('./filter-refs')
-const flattenDeep = require('./flatten-deep')
 const fs = require('fs')
 const { promises: fsp } = fs
 const getCacheDir = require('cache-directory')
@@ -17,11 +18,13 @@ const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
 const globStream = require('glob-stream')
 const invariably = require('./invariably')
+const logger = require('./logger')
 const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')
 const MultiProgress = require('multi-progress') // calls require('progress') as a peer dependencies
 const ospath = require('path')
 const { posix: path } = ospath
-const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
+const posixify = require('./posixify')
+const removeGitSuffix = require('./remove-git-suffix')
 const { fs: resolvePathGlobsFs, git: resolvePathGlobsGit } = require('./resolve-path-globs')
 const { pipeline, Writable } = require('stream')
 const forEach = (write) => new Writable({ objectMode: true, write })
@@ -44,12 +47,9 @@ const {
 const ANY_SEPARATOR_RX = /[:/]/
 const CSV_RX = /\s*,\s*/
 const VENTILATED_CSV_RX = /\s*,\s+/
-const EDIT_URL_TEMPLATE_VAR_RX = /\{(web_url|ref(?:hash|name)|path)\}/g
-const GIT_SUFFIX_RX = /(?:(?:(?:\.git)?\/)?\.git|\/)$/
 const GIT_URI_DETECTOR_RX = /:(?:\/\/|[^/\\])/
-const HEADS_DIR_RX = /^heads\//
-const HOSTED_GIT_REPO_RX = /^(?:https?:\/\/|.+@)(git(?:hub|lab)\.com|bitbucket\.org|pagure\.io)[/:](.+?)(?:\.git)?$/
 const HTTP_ERROR_CODE_RX = new RegExp('^' + git.Errors.HttpError.code + '$', 'i')
+const NEWLINE_RX = /\n/g
 const PATH_SEPARATOR_RX = /[/]/g
 const SHORTEN_REF_RX = /^refs\/(?:heads|remotes\/[^/]+|tags)\//
 const SPACE_RX = / /g
@@ -154,13 +154,14 @@ async function collectFiles (sourcesByUrl, loadOpts, concurrency) {
 
 function buildAggregate (componentVersionBuckets) {
   return [
-    ...flattenDeep(componentVersionBuckets)
+    ...deepFlatten(componentVersionBuckets)
       .reduce((accum, batch) => {
         const key = batch.version + '@' + batch.name
         const entry = accum.get(key)
         if (!entry) return accum.set(key, batch)
-        const files = batch.files
+        const { files, origins } = batch
         ;(batch.files = entry.files).push(...files)
+        ;(batch.origins = entry.origins).push(origins[0])
         Object.assign(entry, batch)
         return accum
       }, new Map())
@@ -257,9 +258,17 @@ function extractCredentials (url) {
 
 async function collectFilesFromSource (source, repo, remoteName, authStatus) {
   const originUrl = repo.url || (await resolveRemoteUrl(repo, remoteName))
-  return selectReferences(source, repo, remoteName).then((refs) =>
-    Promise.all(refs.map((ref) => collectFilesFromReference(source, repo, remoteName, authStatus, ref, originUrl)))
-  )
+  return selectReferences(source, repo, remoteName).then((refs) => {
+    if (!refs.length) {
+      const { url, branches, tags, startPath, startPaths } = source
+      const startPathInfo =
+        'startPaths' in source ? { 'start paths': startPaths || undefined } : { 'start path': startPath || undefined }
+      const sourceInfo = yaml.dump({ url, branches, tags, ...startPathInfo }, { flowLevel: 1 }).trimRight()
+      logger.info(`No matching references found for content source entry (${sourceInfo.replace(NEWLINE_RX, ' | ')})`)
+      return []
+    }
+    return Promise.all(refs.map((it) => collectFilesFromReference(source, repo, remoteName, authStatus, it, originUrl)))
+  })
 }
 
 // QUESTION should we resolve HEAD to a ref eagerly to avoid having to do a match on it?
@@ -267,7 +276,7 @@ async function selectReferences (source, repo, remote) {
   let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns = '.' } = source
   const isBare = repo.noCheckout
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
-  const noWorktree = repo.url ? undefined : null
+  const noWorktree = repo.url ? undefined : false
   const refs = new Map()
   if (
     tagPatterns &&
@@ -403,8 +412,9 @@ async function collectFilesFromReference (source, repo, remoteName, authStatus,
       ? resolvePathGlobsFs(worktreePath, startPaths)
       : resolvePathGlobsGit(repo, ref.oid, startPaths))
     if (!startPaths.length) {
-      const refInfo = `ref: ${ref.fullname.replace(HEADS_DIR_RX, '')}${worktreePath ? ' <worktree>' : ''}`
-      throw new Error(`no start paths found in ${displayUrl} (${refInfo})`)
+      const where = worktreePath || (worktreePath === false ? repo.gitdir : displayUrl)
+      const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
+      throw new Error(`no start paths found in ${where} (${ref.type}: ${ref.shortname}${flag})`)
     }
     return Promise.all(
       startPaths.map((startPath) =>
@@ -417,29 +427,30 @@ async function collectFilesFromReference (source, repo, remoteName, authStatus,
 }
 
 function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version) {
-  return (
-    worktreePath ? readFilesFromWorktree(worktreePath, startPath) : readFilesFromGitTree(repo, ref.oid, startPath)
-  )
-    .then((files) => {
-      const componentVersionBucket = loadComponentDescriptor(files, ref, version)
-      const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
-      componentVersionBucket.files = files.map((file) => assignFileProperties(file, origin))
-      return componentVersionBucket
-    })
+  const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
+  return (worktreePath ? readFilesFromWorktree(origin) : readFilesFromGitTree(repo, ref.oid, startPath))
+    .then((files) =>
+      Object.assign(deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version))), {
+        files: files.map((file) => assignFileProperties(file, origin)),
+        origins: [origin],
+      })
+    )
     .catch((err) => {
-      const msg = err.message
-      const refInfo = `ref: ${ref.fullname.replace(HEADS_DIR_RX, '')}${worktreePath ? ' <worktree>' : ''}`
-      const pathInfo = !startPath || msg.startsWith('the start path ') ? '' : ' | path: ' + startPath
-      throw Object.assign(err, { message: msg.replace(/$/m, ` in ${repo.url || repo.dir} (${refInfo}${pathInfo})`) })
+      const where = worktreePath || (worktreePath === false ? repo.gitdir : repo.url || repo.dir)
+      const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
+      const pathInfo = startPath ? (err.message.startsWith('the start path ') ? '' : ` | start path: ${startPath}`) : ''
+      const message = err.message.replace(/$/m, ` in ${where} (${ref.type}: ${ref.shortname}${flag}${pathInfo})`)
+      throw Object.assign(err, { message })
     })
 }
 
-function readFilesFromWorktree (worktreePath, startPath) {
-  const cwd = ospath.join(worktreePath, startPath, '.') // . shaves off trailing slash
+function readFilesFromWorktree (origin) {
+  const startPath = origin.startPath
+  const cwd = ospath.join(origin.worktree, startPath, '.') // . shaves off trailing slash
   return fsp.stat(cwd).then(
     (startPathStat) => {
       if (!startPathStat.isDirectory()) throw new Error(`the start path '${startPath}' is not a directory`)
-      return srcFs(cwd)
+      return srcFs(cwd, origin)
     },
     () => {
       throw new Error(`the start path '${startPath}' does not exist`)
@@ -447,7 +458,7 @@ function readFilesFromWorktree (worktreePath, startPath) {
   )
 }
 
-function srcFs (cwd) {
+function srcFs (cwd, origin) {
   const relpathStart = cwd.length + 1
   return new Promise((resolve, reject, cache = Object.create(null), files = []) =>
     pipeline(
@@ -465,20 +476,28 @@ function srcFs (cwd) {
                 done()
               },
               (readErr) => {
-                done(Object.assign(readErr, { message: readErr.message.replace(`'${abspath}'`, relpath) }))
+                const logObject = { file: { abspath, origin } }
+                readErr.code === 'ENOENT'
+                  ? logger.warn(logObject, `ENOENT: file or directory disappeared, ${readErr.syscall} ${relpath}`)
+                  : logger.error(logObject, readErr.message.replace(`'${abspath}'`, relpath))
+                done()
               }
             )
           },
           (statErr) => {
+            const logObject = { file: { abspath, origin } }
             if (statErr.symlink) {
-              statErr.message =
-                statErr.code === 'ELOOP'
-                  ? `Symbolic link cycle detected at ${relpath}`
-                  : `Broken symbolic link detected at ${relpath}`
+              logger.error(
+                logObject,
+                (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
+                  `${relpath} -> ${statErr.symlink}`
+              )
+            } else if (statErr.code === 'ENOENT') {
+              logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
             } else {
-              statErr.message = statErr.message.replace(`'${abspath}'`, relpath)
+              logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
             }
-            done(statErr)
+            done()
           }
         )
       }),
@@ -510,30 +529,26 @@ function getGitTreeAtStartPath (repo, oid, startPath) {
 function srcGitTree (repo, root, start) {
   return new Promise((resolve, reject) => {
     const files = []
-    createGitTreeWalker(repo, root, filterGitEntry)
-      .on('entry', (entry) => files.push(entryToFile(entry)))
+    createGitTreeWalker(repo, root, filterGitEntry, gitEntryToFile)
+      .on('entry', (file) => files.push(file))
       .on('error', reject)
-      .on('end', () => resolve(Promise.all(files)))
+      .on('end', () => resolve(files))
       .walk(start)
   })
 }
 
-function createGitTreeWalker (repo, root, filter) {
+function createGitTreeWalker (repo, root, filter, convert) {
   return Object.assign(new EventEmitter(), {
     walk (start) {
-      return (
-        visitGitTree(this, repo, root, filter, start)
-          // NOTE if error is thrown, promises already being resolved won't halt
-          .then(
-            () => this.emit('end'),
-            (err) => this.emit('error', err)
-          )
+      return visitGitTree(this, repo, root, filter, convert, start).then(
+        () => this.emit('end'),
+        (err) => this.emit('error', err)
       )
     },
   })
 }
 
-function visitGitTree (emitter, repo, root, filter, parent, dirname = '', following = new Set()) {
+function visitGitTree (emitter, repo, root, filter, convert, parent, dirname = '', following = new Set()) {
   const reads = []
   for (const entry of parent.tree) {
     const filterVerdict = filter(entry)
@@ -543,7 +558,7 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
         reads.push(
           git.readTree(Object.assign({ oid: entry.oid }, repo)).then((subtree) => {
             Object.assign(subtree, { dirname: path.join(parent.dirname, entry.path) })
-            return visitGitTree(emitter, repo, root, filter, subtree, vfilePath, following)
+            return visitGitTree(emitter, repo, root, filter, convert, subtree, vfilePath, following)
           })
         )
       } else if (entry.type === 'blob') {
@@ -553,56 +568,70 @@ function visitGitTree (emitter, repo, root, filter, parent, dirname = '', follow
             readGitSymlink(repo, root, parent, entry, following).then(
               (target) => {
                 if (target.type === 'tree') {
-                  return visitGitTree(emitter, repo, root, filter, target, vfilePath, new Set(following).add(entry.oid))
+                  return visitGitTree(emitter, repo, root, filter, convert, target, vfilePath, target.following)
                 } else if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
-                  emitter.emit('entry', Object.assign({ mode, oid: target.oid, path: vfilePath }, repo))
+                  return convert(Object.assign({ mode, oid: target.oid, path: vfilePath }, repo)).then((result) =>
+                    emitter.emit('entry', result)
+                  )
                 }
               },
               (err) => {
-                // NOTE this error could be caught after promise chain has already been rejected
-                if (err instanceof NotFoundError) {
-                  err.message = `Broken symbolic link detected at ${vfilePath}`
-                } else if (err.code === 'SymbolicLinkCycleError') {
-                  err.message = `Symbolic link cycle detected at ${vfilePath}`
+                if (err.symlink) {
+                  err.message =
+                    (err.code === 'ELOOP' ? 'ELOOP: symbolic link cycle' : 'ENOENT: broken symbolic link') +
+                    `, ${vfilePath} -> ${err.symlink}`
                 }
                 throw err
               }
             )
           )
         } else if ((mode = FILE_MODES[entry.mode])) {
-          emitter.emit('entry', Object.assign({ mode, oid: entry.oid, path: vfilePath }, repo))
+          reads.push(
+            convert(Object.assign({ mode, oid: entry.oid, path: vfilePath }, repo)).then((result) =>
+              emitter.emit('entry', result)
+            )
+          )
         }
       }
     }
   }
-  return Promise.all(reads)
+  // NOTE preserve scan order so error for symbolic link cycle is deterministic; ensures no rejections after resolve
+  return Promise.allSettled(reads).then((results) => {
+    const rejected = results.find(({ reason }) => reason)
+    if (rejected) throw rejected.reason
+  })
 }
 
-function readGitSymlink (repo, root, parent, { oid }, following) {
-  if (following.size !== (following = new Set(following).add(oid)).size) {
-    return git.readBlob(Object.assign({ oid }, repo)).then(({ blob: target }) => {
-      target = decodeUint8Array(target)
-      let targetParent
-      if (parent.dirname) {
-        const dirname = parent.dirname + '/'
-        target = path.join(dirname, target) // join doesn't remove trailing separator
-        if (target.startsWith(dirname)) {
-          target = target.substr(dirname.length)
-          targetParent = parent
-        } else {
-          targetParent = root
-        }
-      } else {
-        target = path.normalize(target) // normalize doesn't remove trailing separator
-        targetParent = root
+function readGitSymlink (repo, root, parent, { oid, path: name }, following) {
+  const dirname = parent.dirname
+  if (following.size === (following = new Set(following)).add(oid).size) {
+    const err = { name: 'SymbolicLinkCycleError', code: 'ELOOP', oid, path: `${path.join(dirname, name)}` }
+    return Promise.reject(Object.assign(new Error(`Symbolic link cycle detected at ${oid}:${err.path}`), err))
+  }
+  return git.readBlob(Object.assign({ oid }, repo)).then(({ blob: symlink }) => {
+    symlink = decodeUint8Array(symlink)
+    let target
+    let targetParent = root
+    if (dirname) {
+      if (!(target = path.join('/', dirname, symlink).substr(1)) || target === dirname) {
+        target = '.'
+      } else if (target.startsWith(dirname + '/')) {
+        target = target.substr(dirname.length + 1) // join doesn't remove trailing separator
+        targetParent = parent
       }
-      const targetSegments = target.split('/')
-      if (!targetSegments[targetSegments.length - 1]) targetSegments.pop()
-      return readGitObjectAtPath(repo, root, targetParent, targetSegments, following)
+    } else {
+      target = path.normalize(symlink) // normalize doesn't remove trailing separator
+    }
+    if (target === '.') {
+      const err = { name: 'SymbolicLinkCycleError', code: 'ELOOP', oid, path: `${path.join(dirname, name)}`, symlink }
+      return Promise.reject(Object.assign(new Error(`Symbolic link cycle detected at ${oid}:${err.path}`), err))
+    }
+    const targetSegments = target.split('/')
+    targetSegments[targetSegments.length - 1] || targetSegments.pop()
+    return readGitObjectAtPath(repo, root, targetParent, targetSegments, following).catch((err) => {
+      throw Object.assign(err, { symlink })
     })
-  }
-  const err = { name: 'SymbolicLinkCycleError', code: 'SymbolicLinkCycleError', oid }
-  return Promise.reject(Object.assign(new Error(`Symbolic link cycle found at oid: ${err.oid}`), err))
+  })
 }
 
 // QUESTION: could we use this to resolve the start path too?
@@ -615,14 +644,14 @@ function readGitObjectAtPath (repo, root, parent, pathSegments, following) {
           Object.assign(subtree, { dirname: path.join(parent.dirname, entry.path) })
           return (pathSegments = pathSegments.slice(1)).length
             ? readGitObjectAtPath(repo, root, subtree, pathSegments, following)
-            : Object.assign(subtree, { type: 'tree' })
+            : Object.assign(subtree, { type: 'tree', following }) // Q: should this create copy?
         })
         : entry.mode === SYMLINK_FILE_MODE
           ? readGitSymlink(repo, root, parent, entry, following)
           : Promise.resolve(entry)
     }
   }
-  return Promise.reject(new NotFoundError(`No file or directory found at "${parent.oid}:${pathSegments.join('/')}"`))
+  return Promise.reject(new NotFoundError(`${parent.oid}:${pathSegments.join('/')}`))
 }
 
 /**
@@ -637,7 +666,7 @@ function filterGitEntry (entry) {
   return entryPath.charAt(entryPath.length - 1) !== '~'
 }
 
-function entryToFile (entry) {
+function gitEntryToFile (entry) {
   return git.readBlob(entry).then(({ blob: contents }) => {
     contents = Buffer.from(contents.buffer)
     const stat = Object.assign(new fs.Stats(), { mode: entry.mode, mtime: undefined, size: contents.byteLength })
@@ -691,53 +720,7 @@ function loadComponentDescriptor (files, ref, version) {
     throw new Error(`version in ${COMPONENT_DESC_FILENAME} cannot have path segments: ${version}`)
   }
   data.version = version
-  return camelCaseKeys(data, { deep: true, stopPaths: ['asciidoc'] })
-}
-
-function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath = undefined, editUrl = true) {
-  const { shortname: refname, oid: refhash, type: reftype } = ref
-  const origin = { type: 'git', url, gitdir, refname, [reftype]: refname, startPath }
-  if (authStatus) origin.private = authStatus
-  if (worktreePath === undefined) {
-    origin.refhash = refhash
-  } else {
-    if (worktreePath) {
-      origin.fileUriPattern =
-        (posixify ? 'file:///' + posixify(worktreePath) : 'file://' + worktreePath) +
-        (startPath ? '/' + startPath + '/%s' : '/%s')
-    } else {
-      origin.refhash = refhash
-    }
-    origin.worktree = worktreePath
-    if (url.startsWith('file://')) url = undefined
-  }
-  if (url) origin.webUrl = url.replace(GIT_SUFFIX_RX, '')
-  if (editUrl === true) {
-    let match
-    if (url && (match = url.match(HOSTED_GIT_REPO_RX))) {
-      const host = match[1]
-      let action
-      let category = ''
-      if (host === 'pagure.io') {
-        action = 'blob'
-        category = 'f'
-      } else if (host === 'bitbucket.org') {
-        action = 'src'
-      } else {
-        action = reftype === 'branch' ? 'edit' : 'blob'
-      }
-      origin.editUrlPattern = 'https://' + path.join(match[1], match[2], action, refname, category, startPath, '%s')
-    }
-  } else if (editUrl) {
-    const vars = {
-      path: () => (startPath ? path.join(startPath, '%s') : '%s'),
-      refhash: () => refhash,
-      refname: () => refname,
-      web_url: () => origin.webUrl || '',
-    }
-    origin.editUrlPattern = editUrl.replace(EDIT_URL_TEMPLATE_VAR_RX, (_, name) => vars[name]())
-  }
-  return origin
+  return camelCaseKeys(data, ['asciidoc'])
 }
 
 function assignFileProperties (file, origin) {
@@ -823,17 +806,17 @@ function onGitProgress ({ phase, loaded, total }) {
     const scaleFactor = this.scaleFactor
     let ratio = ((loaded / total) * scaleFactor) / GIT_PROGRESS_PHASES.length
     if (phaseIdx) ratio += (phaseIdx * scaleFactor) / GIT_PROGRESS_PHASES.length
-    // NOTE: updates are automatically throttled based on renderThrottle option
+    // NOTE updates are automatically throttled based on renderThrottle option
     this.update(ratio > scaleFactor ? scaleFactor : ratio)
   }
 }
 
 function onGitComplete (err) {
   if (err) {
-    // TODO: could use progressBar.interrupt() to replace bar with message instead
+    // TODO could use progressBar.interrupt() to replace bar with message instead
     this.chars.incomplete = '?'
     this.update(0)
-    // NOTE: force progress bar to update regardless of throttle setting
+    // NOTE force progress bar to update regardless of throttle setting
     this.render(undefined, true)
   } else {
     this.update(1)
@@ -871,7 +854,7 @@ function resolveCredentials (credentialsFromUrlHolder, url, auth) {
 function generateCloneFolderName (url) {
   let normalizedUrl = url.toLowerCase()
   if (posixify) normalizedUrl = posixify(normalizedUrl)
-  normalizedUrl = normalizedUrl.replace(GIT_SUFFIX_RX, '')
+  normalizedUrl = removeGitSuffix(normalizedUrl)
   const basename = normalizedUrl.split(ANY_SEPARATOR_RX).pop()
   const hash = createHash('sha1')
   hash.update(normalizedUrl)
@@ -914,9 +897,14 @@ function isDirectory (url) {
 function symlinkAwareStat (path_) {
   return fsp.lstat(path_).then((lstat) => {
     if (!lstat.isSymbolicLink()) return lstat
-    return fsp.stat(path_).catch((statErr) => {
-      throw Object.assign(statErr, { symlink: true })
-    })
+    return fsp.stat(path_).catch((statErr) =>
+      fsp
+        .readlink(path_)
+        .catch(invariably.void)
+        .then((symlink) => {
+          throw Object.assign(statErr, { symlink })
+        })
+    )
   })
 }
 
@@ -994,23 +982,35 @@ function transformGitCloneError (err, displayUrl) {
   if (trimMessage) {
     wrappedMsg = ~(wrappedMsg = wrappedMsg.trimRight()).indexOf('. ') ? wrappedMsg : wrappedMsg.replace(/\.$/, '')
   }
-  const wrappedErr = new Error(`${wrappedMsg} (url: ${displayUrl})`)
-  wrappedErr.stack += `\nCaused by: ${err.stack || 'unknown'}`
-  return wrappedErr
+  const errWrapper = new Error(`${wrappedMsg} (url: ${displayUrl})`)
+  errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
+  return errWrapper
 }
 
 function splitRefPatterns (str) {
   return ~str.indexOf('{') ? str.split(VENTILATED_CSV_RX) : str.split(CSV_RX)
 }
 
-function coerceToString (value) {
-  return value == null ? '' : String(value)
+function camelCaseKeys (o, stopPaths = [], p = '') {
+  if (Array.isArray(o)) return o.map((it) => camelCaseKeys(it, stopPaths, p))
+  if (o == null || o.constructor !== Object) return o
+  const pathPrefix = p && p + '.'
+  const accum = {}
+  for (const [k, v] of Object.entries(o)) {
+    const camelKey = k.toLowerCase().replace(/[_-]([a-z0-9])/g, (_, l, idx) => (idx ? l.toUpperCase() : l))
+    accum[camelKey] = ~stopPaths.indexOf(pathPrefix + camelKey) ? v : camelCaseKeys(v, stopPaths, pathPrefix + camelKey)
+  }
+  return accum
 }
 
 function cleanStartPath (value) {
   return value && ~value.indexOf('/') ? value.replace(SUPERFLUOUS_SEPARATORS_RX, '') : value
 }
 
+function coerceToString (value) {
+  return value == null ? '' : String(value)
+}
+
 function findWorktrees (repo, patterns) {
   if (!patterns.length) return new Map()
   const linkedOnly = patterns[0] === '.' ? !(patterns = patterns.slice(1)) : true
@@ -1047,4 +1047,3 @@ function findWorktrees (repo, patterns) {
 }
 
 module.exports = aggregateContent
-module.exports._computeOrigin = computeOrigin

package/lib/compute-origin.js

@@ -0,0 +1,54 @@
+'use strict'
+
+const { posix: path } = require('path')
+const posixify = require('./posixify')
+const removeGitSuffix = require('./remove-git-suffix')
+
+const EDIT_URL_TEMPLATE_VAR_RX = /\{(web_url|ref(?:hash|name|type)|path)\}/g
+const HOSTED_GIT_REPO_RX = /^(?:https?:\/\/|.+@)(git(?:hub|lab)\.com|bitbucket\.org|pagure\.io)[/:](.+?)(?:\.git)?$/
+
+function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath = undefined, editUrl = true) {
+  const { shortname: refname, oid: refhash, remote, type: reftype } = ref
+  const origin = { type: 'git', url, gitdir, reftype, refname, [reftype]: refname, refhash, startPath }
+  if (worktreePath !== undefined) {
+    if ((origin.worktree = worktreePath)) {
+      delete origin.refhash
+      origin.fileUriPattern =
+        (posixify ? 'file:///' + posixify(worktreePath) : 'file://' + worktreePath) +
+        (startPath ? '/' + startPath + '/%s' : '/%s')
+    } else if (remote) {
+      origin.remote = remote
+    }
+    if (url.startsWith('file://')) url = undefined
+  }
+  if (authStatus) origin.private = authStatus
+  if (url) origin.webUrl = removeGitSuffix(url)
+  if (editUrl === true) {
+    const match = url && url.match(HOSTED_GIT_REPO_RX)
+    if (match) {
+      const host = match[1]
+      let action = 'blob'
+      let category = ''
+      if (host === 'pagure.io') {
+        category = 'f'
+      } else if (host === 'bitbucket.org') {
+        action = 'src'
+      } else if (reftype === 'branch') {
+        action = 'edit'
+      }
+      origin.editUrlPattern = 'https://' + path.join(match[1], match[2], action, refname, category, startPath, '%s')
+    }
+  } else if (editUrl) {
+    const vars = {
+      path: () => (startPath ? path.join(startPath, '%s') : '%s'),
+      refhash: () => refhash,
+      reftype: () => reftype,
+      refname: () => refname,
+      web_url: () => origin.webUrl || '',
+    }
+    origin.editUrlPattern = editUrl.replace(EDIT_URL_TEMPLATE_VAR_RX, (_, name) => vars[name]())
+  }
+  return origin
+}
+
+module.exports = computeOrigin

package/lib/deep-clone.js

@@ -0,0 +1,18 @@
+'use strict'
+
+function deepClone (o) {
+  switch (o.constructor) {
+    case Object:
+      return Object.keys(o).reduce((accum, k) => {
+        const v = o[k]
+        accum[k] = !v || typeof v !== 'object' ? v : deepClone(v)
+        return accum
+      }, {})
+    case Array:
+      return o.map((it) => (!it || typeof it !== 'object' ? it : deepClone(it)))
+    default:
+      return o
+  }
+}
+
+module.exports = deepClone

package/lib/{flatten-deep.js → deep-flatten.js}

@@ -1,9 +1,9 @@
 'use strict'
 
-function flattenDeep (array, accum = []) {
+function deepFlatten (array, accum = []) {
   const len = array.length
-  for (let i = 0, it; i < len; i++) Array.isArray((it = array[i])) ? flattenDeep(it, accum) : accum.push(it)
+  for (let i = 0, it; i < len; i++) Array.isArray((it = array[i])) ? deepFlatten(it, accum) : accum.push(it)
   return accum
 }
 
-module.exports = flattenDeep
+module.exports = deepFlatten

package/lib/git-plugin-http.js

@@ -12,18 +12,12 @@ async function mergeBuffers (data) {
   if (!Array.isArray(data)) return data
   if (data.length === 1 && data[0] instanceof Buffer) return data[0]
   const buffers = []
-  let offset = 0
-  let size = 0
+  let totalLength = 0
   for await (const chunk of data) {
     buffers.push(chunk)
-    size += chunk.byteLength
+    totalLength += chunk.byteLength
   }
-  data = new Uint8Array(size)
-  for (const buffer of buffers) {
-    data.set(buffer, offset)
-    offset += buffer.byteLength
-  }
-  return Buffer.from(data.buffer)
+  return Buffer.concat(buffers, totalLength)
 }
 
 function mergeHeaders (headers, extraHeaders) {

package/lib/logger.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('@antora/logger')(require('../package.json').name)

package/lib/matcher.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { compile: bracesToGroup, expand: expandBraces } = require('braces')
-const { makeRe: makeMatcherRx } = require('picomatch')
+const { makeRe } = require('picomatch')
 
 const BASE_OPTS = {
   bash: true,
@@ -16,6 +16,11 @@ const BASE_OPTS = {
   strictSlashes: true,
 }
 
+function makeMatcherRx (input, opts) {
+  if (input && ~input.indexOf('{')) input = input.replace(/^([^({]+)\./, '$1(?:.)')
+  return makeRe(input, opts)
+}
+
 module.exports = {
   MATCH_ALL_RX: { test: () => true },
   expandBraces,

package/lib/posixify.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('path').sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined

package/lib/remove-git-suffix.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const GIT_SUFFIX_RX = /(?:(?:(?:\.git)?\/)?\.git|\/)$/
+
+function removeGitSuffix (url) {
+  return url.replace(GIT_SUFFIX_RX, '')
+}
+
+module.exports = removeGitSuffix

package/lib/resolve-path-globs.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const flattenDeep = require('./flatten-deep')
+const deepFlatten = require('./deep-flatten')
 const { promises: fsp } = require('fs')
 const git = require('./git')
 const invariably = require('./invariably')
@@ -56,7 +56,7 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
     }
     let dirents = await listDirents(base, oid || path)
     if (explicit) dirents = dirents.filter((dirent) => !explicit.has(dirent.name))
-    const discovered = flattenDeep(
+    const discovered = deepFlatten(
       await Promise.all(
         dirents.map((dirent) =>
           dirent.isDirectory() && isMatch(dirent.name)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.0.1",
+  "version": "3.1.0",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -16,18 +16,26 @@
     "url": "https://gitlab.com/antora/antora/issues"
   },
   "main": "lib/index.js",
-  "scripts": {
-    "test": "_mocha"
+  "exports": {
+    ".": "./lib/index.js",
+    "./git": "./lib/git.js",
+    "./git/http-plugin": "./lib/git-plugin-http.js",
+    "./lib/git-plugin-http": "./lib/git-plugin-http.js",
+    "./package.json": "./package.json"
+  },
+  "imports": {
+    "#compute-origin": "./lib/compute-origin.js",
+    "#constants": "./lib/constants.js"
   },
   "dependencies": {
     "@antora/expand-path-helper": "~2.0",
+    "@antora/logger": "3.1.0",
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~7.0",
     "glob-stream": "~7.0",
-    "hpagent": "~0.1.0",
-    "isomorphic-git": "~1.10",
+    "hpagent": "~1.0",
+    "isomorphic-git": "~1.19",
     "js-yaml": "~4.1",
     "multi-progress": "~4.0",
     "picomatch": "~2.3",
@@ -37,7 +45,7 @@
     "vinyl": "~2.2"
   },
   "engines": {
-    "node": ">=12.21.0"
+    "node": ">=16.0.0"
   },
   "files": [
     "lib/"
@@ -52,5 +60,9 @@
     "static site",
     "web publishing"
   ],
-  "gitHead": "e8e6f6ba33b1ab3f796907b5a256893a64844cd1"
+  "scripts": {
+    "test": "_mocha",
+    "prepublishOnly": "node $npm_config_local_prefix/npm/prepublishOnly.js",
+    "postpublish": "node $npm_config_local_prefix/npm/postpublish.js"
+  }
 }

package/LICENSE

@@ -1,373 +0,0 @@
-Mozilla Public License Version 2.0
-==================================
-
-1. Definitions
---------------
-
-1.1. "Contributor"
-    means each individual or legal entity that creates, contributes to
-    the creation of, or owns Covered Software.
-
-1.2. "Contributor Version"
-    means the combination of the Contributions of others (if any) used
-    by a Contributor and that particular Contributor's Contribution.
-
-1.3. "Contribution"
-    means Covered Software of a particular Contributor.
-
-1.4. "Covered Software"
-    means Source Code Form to which the initial Contributor has attached
-    the notice in Exhibit A, the Executable Form of such Source Code
-    Form, and Modifications of such Source Code Form, in each case
-    including portions thereof.
-
-1.5. "Incompatible With Secondary Licenses"
-    means
-
-    (a) that the initial Contributor has attached the notice described
-        in Exhibit B to the Covered Software; or
-
-    (b) that the Covered Software was made available under the terms of
-        version 1.1 or earlier of the License, but not also under the
-        terms of a Secondary License.
-
-1.6. "Executable Form"
-    means any form of the work other than Source Code Form.
-
-1.7. "Larger Work"
-    means a work that combines Covered Software with other material, in
-    a separate file or files, that is not Covered Software.
-
-1.8. "License"
-    means this document.
-
-1.9. "Licensable"
-    means having the right to grant, to the maximum extent possible,
-    whether at the time of the initial grant or subsequently, any and
-    all of the rights conveyed by this License.
-
-1.10. "Modifications"
-    means any of the following:
-
-    (a) any file in Source Code Form that results from an addition to,
-        deletion from, or modification of the contents of Covered
-        Software; or
-
-    (b) any new file in Source Code Form that contains any Covered
-        Software.
-
-1.11. "Patent Claims" of a Contributor
-    means any patent claim(s), including without limitation, method,
-    process, and apparatus claims, in any patent Licensable by such
-    Contributor that would be infringed, but for the grant of the
-    License, by the making, using, selling, offering for sale, having
-    made, import, or transfer of either its Contributions or its
-    Contributor Version.
-
-1.12. "Secondary License"
-    means either the GNU General Public License, Version 2.0, the GNU
-    Lesser General Public License, Version 2.1, the GNU Affero General
-    Public License, Version 3.0, or any later versions of those
-    licenses.
-
-1.13. "Source Code Form"
-    means the form of the work preferred for making modifications.
-
-1.14. "You" (or "Your")
-    means an individual or a legal entity exercising rights under this
-    License. For legal entities, "You" includes any entity that
-    controls, is controlled by, or is under common control with You. For
-    purposes of this definition, "control" means (a) the power, direct
-    or indirect, to cause the direction or management of such entity,
-    whether by contract or otherwise, or (b) ownership of more than
-    fifty percent (50%) of the outstanding shares or beneficial
-    ownership of such entity.
-
-2. License Grants and Conditions
---------------------------------
-
-2.1. Grants
-
-Each Contributor hereby grants You a world-wide, royalty-free,
-non-exclusive license:
-
-(a) under intellectual property rights (other than patent or trademark)
-    Licensable by such Contributor to use, reproduce, make available,
-    modify, display, perform, distribute, and otherwise exploit its
-    Contributions, either on an unmodified basis, with Modifications, or
-    as part of a Larger Work; and
-
-(b) under Patent Claims of such Contributor to make, use, sell, offer
-    for sale, have made, import, and otherwise transfer either its
-    Contributions or its Contributor Version.
-
-2.2. Effective Date
-
-The licenses granted in Section 2.1 with respect to any Contribution
-become effective for each Contribution on the date the Contributor first
-distributes such Contribution.
-
-2.3. Limitations on Grant Scope
-
-The licenses granted in this Section 2 are the only rights granted under
-this License. No additional rights or licenses will be implied from the
-distribution or licensing of Covered Software under this License.
-Notwithstanding Section 2.1(b) above, no patent license is granted by a
-Contributor:
-
-(a) for any code that a Contributor has removed from Covered Software;
-    or
-
-(b) for infringements caused by: (i) Your and any other third party's
-    modifications of Covered Software, or (ii) the combination of its
-    Contributions with other software (except as part of its Contributor
-    Version); or
-
-(c) under Patent Claims infringed by Covered Software in the absence of
-    its Contributions.
-
-This License does not grant any rights in the trademarks, service marks,
-or logos of any Contributor (except as may be necessary to comply with
-the notice requirements in Section 3.4).
-
-2.4. Subsequent Licenses
-
-No Contributor makes additional grants as a result of Your choice to
-distribute the Covered Software under a subsequent version of this
-License (see Section 10.2) or under the terms of a Secondary License (if
-permitted under the terms of Section 3.3).
-
-2.5. Representation
-
-Each Contributor represents that the Contributor believes its
-Contributions are its original creation(s) or it has sufficient rights
-to grant the rights to its Contributions conveyed by this License.
-
-2.6. Fair Use
-
-This License is not intended to limit any rights You have under
-applicable copyright doctrines of fair use, fair dealing, or other
-equivalents.
-
-2.7. Conditions
-
-Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
-in Section 2.1.
-
-3. Responsibilities
--------------------
-
-3.1. Distribution of Source Form
-
-All distribution of Covered Software in Source Code Form, including any
-Modifications that You create or to which You contribute, must be under
-the terms of this License. You must inform recipients that the Source
-Code Form of the Covered Software is governed by the terms of this
-License, and how they can obtain a copy of this License. You may not
-attempt to alter or restrict the recipients' rights in the Source Code
-Form.
-
-3.2. Distribution of Executable Form
-
-If You distribute Covered Software in Executable Form then:
-
-(a) such Covered Software must also be made available in Source Code
-    Form, as described in Section 3.1, and You must inform recipients of
-    the Executable Form how they can obtain a copy of such Source Code
-    Form by reasonable means in a timely manner, at a charge no more
-    than the cost of distribution to the recipient; and
-
-(b) You may distribute such Executable Form under the terms of this
-    License, or sublicense it under different terms, provided that the
-    license for the Executable Form does not attempt to limit or alter
-    the recipients' rights in the Source Code Form under this License.
-
-3.3. Distribution of a Larger Work
-
-You may create and distribute a Larger Work under terms of Your choice,
-provided that You also comply with the requirements of this License for
-the Covered Software. If the Larger Work is a combination of Covered
-Software with a work governed by one or more Secondary Licenses, and the
-Covered Software is not Incompatible With Secondary Licenses, this
-License permits You to additionally distribute such Covered Software
-under the terms of such Secondary License(s), so that the recipient of
-the Larger Work may, at their option, further distribute the Covered
-Software under the terms of either this License or such Secondary
-License(s).
-
-3.4. Notices
-
-You may not remove or alter the substance of any license notices
-(including copyright notices, patent notices, disclaimers of warranty,
-or limitations of liability) contained within the Source Code Form of
-the Covered Software, except that You may alter any license notices to
-the extent required to remedy known factual inaccuracies.
-
-3.5. Application of Additional Terms
-
-You may choose to offer, and to charge a fee for, warranty, support,
-indemnity or liability obligations to one or more recipients of Covered
-Software. However, You may do so only on Your own behalf, and not on
-behalf of any Contributor. You must make it absolutely clear that any
-such warranty, support, indemnity, or liability obligation is offered by
-You alone, and You hereby agree to indemnify every Contributor for any
-liability incurred by such Contributor as a result of warranty, support,
-indemnity or liability terms You offer. You may include additional
-disclaimers of warranty and limitations of liability specific to any
-jurisdiction.
-
-4. Inability to Comply Due to Statute or Regulation
----------------------------------------------------
-
-If it is impossible for You to comply with any of the terms of this
-License with respect to some or all of the Covered Software due to
-statute, judicial order, or regulation then You must: (a) comply with
-the terms of this License to the maximum extent possible; and (b)
-describe the limitations and the code they affect. Such description must
-be placed in a text file included with all distributions of the Covered
-Software under this License. Except to the extent prohibited by statute
-or regulation, such description must be sufficiently detailed for a
-recipient of ordinary skill to be able to understand it.
-
-5. Termination
---------------
-
-5.1. The rights granted under this License will terminate automatically
-if You fail to comply with any of its terms. However, if You become
-compliant, then the rights granted under this License from a particular
-Contributor are reinstated (a) provisionally, unless and until such
-Contributor explicitly and finally terminates Your grants, and (b) on an
-ongoing basis, if such Contributor fails to notify You of the
-non-compliance by some reasonable means prior to 60 days after You have
-come back into compliance. Moreover, Your grants from a particular
-Contributor are reinstated on an ongoing basis if such Contributor
-notifies You of the non-compliance by some reasonable means, this is the
-first time You have received notice of non-compliance with this License
-from such Contributor, and You become compliant prior to 30 days after
-Your receipt of the notice.
-
-5.2. If You initiate litigation against any entity by asserting a patent
-infringement claim (excluding declaratory judgment actions,
-counter-claims, and cross-claims) alleging that a Contributor Version
-directly or indirectly infringes any patent, then the rights granted to
-You by any and all Contributors for the Covered Software under Section
-2.1 of this License shall terminate.
-
-5.3. In the event of termination under Sections 5.1 or 5.2 above, all
-end user license agreements (excluding distributors and resellers) which
-have been validly granted by You or Your distributors under this License
-prior to termination shall survive termination.
-
-************************************************************************
-*                                                                      *
-*  6. Disclaimer of Warranty                                           *
-*  -------------------------                                           *
-*                                                                      *
-*  Covered Software is provided under this License on an "as is"       *
-*  basis, without warranty of any kind, either expressed, implied, or  *
-*  statutory, including, without limitation, warranties that the       *
-*  Covered Software is free of defects, merchantable, fit for a        *
-*  particular purpose or non-infringing. The entire risk as to the     *
-*  quality and performance of the Covered Software is with You.        *
-*  Should any Covered Software prove defective in any respect, You     *
-*  (not any Contributor) assume the cost of any necessary servicing,   *
-*  repair, or correction. This disclaimer of warranty constitutes an   *
-*  essential part of this License. No use of any Covered Software is   *
-*  authorized under this License except under this disclaimer.         *
-*                                                                      *
-************************************************************************
-
-************************************************************************
-*                                                                      *
-*  7. Limitation of Liability                                          *
-*  --------------------------                                          *
-*                                                                      *
-*  Under no circumstances and under no legal theory, whether tort      *
-*  (including negligence), contract, or otherwise, shall any           *
-*  Contributor, or anyone who distributes Covered Software as          *
-*  permitted above, be liable to You for any direct, indirect,         *
-*  special, incidental, or consequential damages of any character      *
-*  including, without limitation, damages for lost profits, loss of    *
-*  goodwill, work stoppage, computer failure or malfunction, or any    *
-*  and all other commercial damages or losses, even if such party      *
-*  shall have been informed of the possibility of such damages. This   *
-*  limitation of liability shall not apply to liability for death or   *
-*  personal injury resulting from such party's negligence to the       *
-*  extent applicable law prohibits such limitation. Some               *
-*  jurisdictions do not allow the exclusion or limitation of           *
-*  incidental or consequential damages, so this exclusion and          *
-*  limitation may not apply to You.                                    *
-*                                                                      *
-************************************************************************
-
-8. Litigation
--------------
-
-Any litigation relating to this License may be brought only in the
-courts of a jurisdiction where the defendant maintains its principal
-place of business and such litigation shall be governed by laws of that
-jurisdiction, without reference to its conflict-of-law provisions.
-Nothing in this Section shall prevent a party's ability to bring
-cross-claims or counter-claims.
-
-9. Miscellaneous
-----------------
-
-This License represents the complete agreement concerning the subject
-matter hereof. If any provision of this License is held to be
-unenforceable, such provision shall be reformed only to the extent
-necessary to make it enforceable. Any law or regulation which provides
-that the language of a contract shall be construed against the drafter
-shall not be used to construe this License against a Contributor.
-
-10. Versions of the License
----------------------------
-
-10.1. New Versions
-
-Mozilla Foundation is the license steward. Except as provided in Section
-10.3, no one other than the license steward has the right to modify or
-publish new versions of this License. Each version will be given a
-distinguishing version number.
-
-10.2. Effect of New Versions
-
-You may distribute the Covered Software under the terms of the version
-of the License under which You originally received the Covered Software,
-or under the terms of any subsequent version published by the license
-steward.
-
-10.3. Modified Versions
-
-If you create software not governed by this License, and you want to
-create a new license for such software, you may create and use a
-modified version of this License if you rename the license and remove
-any references to the name of the license steward (except to note that
-such modified license differs from this License).
-
-10.4. Distributing Source Code Form that is Incompatible With Secondary
-Licenses
-
-If You choose to distribute Source Code Form that is Incompatible With
-Secondary Licenses under the terms of this version of the License, the
-notice described in Exhibit B of this License must be attached.
-
-Exhibit A - Source Code Form License Notice
--------------------------------------------
-
-  This Source Code Form is subject to the terms of the Mozilla Public
-  License, v. 2.0. If a copy of the MPL was not distributed with this
-  file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-If it is not possible or desirable to put the notice in a particular
-file, then You may include the notice in a location (such as a LICENSE
-file in a relevant directory) where a recipient would be likely to look
-for such a notice.
-
-You may add additional accurate notices of copyright ownership.
-
-Exhibit B - "Incompatible With Secondary Licenses" Notice
----------------------------------------------------------
-
-  This Source Code Form is "Incompatible With Secondary Licenses", as
-  defined by the Mozilla Public License, v. 2.0.