@antora/content-aggregator 3.2.0-alpha.1 → 3.2.0-alpha.11

package/lib/aggregate-content.js

@@ -1,32 +1,32 @@
 'use strict'
 
 const computeOrigin = require('./compute-origin')
-const { createHash } = require('crypto')
+const { createHash } = require('node:crypto')
 const createGitHttpPlugin = require('./git-plugin-http')
 const decodeUint8Array = require('./decode-uint8-array')
 const deepClone = require('./deep-clone')
-const deepFlatten = require('./deep-flatten')
-const EventEmitter = require('events')
+const EventEmitter = require('node:events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
 const filterRefs = require('./filter-refs')
-const fs = require('fs')
+const fs = require('node:fs')
 const { promises: fsp } = fs
 const getCacheDir = require('cache-directory')
 const GitCredentialManagerStore = require('./git-credential-manager-store')
 const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
-const globStream = require('glob-stream')
+const { globStream } = require('fast-glob')
+const { inspect } = require('node:util')
 const invariably = require('./invariably')
 const logger = require('./logger')
 const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')
 const MultiProgress = require('multi-progress') // calls require('progress') as a peer dependencies
-const ospath = require('path')
+const ospath = require('node:path')
 const { posix: path } = ospath
 const posixify = require('./posixify')
 const removeGitSuffix = require('./remove-git-suffix')
 const { fs: resolvePathGlobsFs, git: resolvePathGlobsGit } = require('./resolve-path-globs')
-const { pipeline, Writable } = require('stream')
+const { pipeline, Writable } = require('node:stream')
 const forEach = (write) => new Writable({ objectMode: true, write })
 const userRequire = require('@antora/user-require-helper')
 const yaml = require('js-yaml')
@@ -87,138 +87,132 @@ const URL_PORT_CLEANER_RX = /^([^/]+):[0-9]+(?=\/)/
  */
 function aggregateContent (playbook) {
   const startDir = playbook.dir || '.'
-  const { branches, editUrl, tags, sources } = playbook.content
-  const sourceDefaults = { branches, editUrl, tags }
+  const { branches, editUrl, tags, sources, worktrees } = playbook.content
+  const sourceDefaults = { branches, editUrl, tags, worktrees }
   const { cacheDir: requestedCacheDir, fetch, quiet } = playbook.runtime
   return ensureCacheDir(requestedCacheDir, startDir).then((cacheDir) => {
     const gitConfig = Object.assign({ ensureGitSuffix: true }, playbook.git)
     const gitPlugins = loadGitPlugins(gitConfig, playbook.network || {}, startDir)
-    const fetchConcurrency = Math.max(gitConfig.fetchConcurrency || Infinity, 1)
+    const concurrency = {
+      fetch: Math.max(gitConfig.fetchConcurrency || Infinity, 1),
+      read: Math.max(gitConfig.readConcurrency || Infinity, 1),
+    }
     const sourcesByUrl = sources.reduce((accum, source) => {
       return accum.set(source.url, [...(accum.get(source.url) || []), Object.assign({}, sourceDefaults, source)])
     }, new Map())
-    const progress = !quiet && createProgress(sourcesByUrl.keys(), process.stdout)
+    const progress = quiet ? undefined : createProgress(sourcesByUrl.keys(), process.stdout)
     const refPatternCache = Object.assign(new Map(), { braces: new Map() })
-    const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir, refPatternCache }
-    return collectFiles(sourcesByUrl, loadOpts, fetchConcurrency).then(buildAggregate, (err) => {
-      progress && progress.terminate()
+    const fetchConfig = { always: fetch, depth: Math.max(0, gitConfig.fetchDepth ?? 1) }
+    const loadOpts = { cacheDir, fetch: fetchConfig, gitPlugins, progress, startDir, refPatternCache }
+    return collectFiles(sourcesByUrl, loadOpts, concurrency).then(buildAggregate, (err) => {
+      progress?.terminate()
       throw err
     })
   })
 }
 
-async function collectFiles (sourcesByUrl, loadOpts, concurrency) {
-  const tasks = [...sourcesByUrl.entries()].map(([url, sources]) => [
-    () => loadRepository(url, Object.assign({ fetchTags: tagsSpecified(sources) }, loadOpts)),
-    ({ repo, authStatus }) =>
-      Promise.all(
-        sources.map((source) => {
-          // NOTE if repository is managed (has a url property), we can assume the remote name is origin
-          // TODO if the repo has no remotes, then remoteName should be undefined
-          const remoteName = repo.url ? 'origin' : source.remote || 'origin'
-          return collectFilesFromSource(source, repo, remoteName, authStatus)
-        })
-      ),
-  ])
-  let rejection, started
-  const startedContinuations = []
-  const recordRejection = (err) => {
-    rejection = err
-  }
-  const runTask = (primary, continuation, idx) =>
-    primary().then((value) => {
-      if (!rejection) startedContinuations[idx] = continuation(value).catch(recordRejection)
-    }, recordRejection)
-  if (tasks.length > concurrency) {
-    started = []
-    const pending = []
-    for (const [primary, continuation] of tasks) {
-      const current = runTask(primary, continuation, started.length).finally(() =>
-        pending.splice(pending.indexOf(current), 1)
-      )
-      started.push(current)
-      if (pending.push(current) < concurrency) continue
-      await Promise.race(pending)
-      if (rejection) break
+async function collectFiles (sourcesByUrl, loadOpts, concurrency, fetchedUrls = []) {
+  const loadTasks = [...sourcesByUrl.entries()].map(([url, sources]) => {
+    const loadOptsForUrl = Object.assign({}, loadOpts)
+    if (loadOpts.fetch.always && fetchedUrls.length && ~fetchedUrls.indexOf(url)) loadOptsForUrl.fetch.always = false
+    if (tagsSpecified(sources)) loadOptsForUrl.fetch.tags = true
+    const commits = commitsRequested(sources)
+    if (commits) loadOptsForUrl.fetch.commits = commits
+    return loadRepository.bind(null, url, loadOptsForUrl, { url, sources })
+  })
+  return gracefulPromiseAllWithLimit(loadTasks, concurrency.fetch).then(([results, rejections]) => {
+    if (rejections.length) {
+      if (concurrency.fetch > 1 && results.length > 1 && rejections.every(({ recoverable }) => recoverable)) {
+        if (loadOpts.progress) loadOpts.progress.terminate() // reset cursor position and allow it be reused
+        const msg0 = 'An unexpected error occurred while fetching content sources concurrently.'
+        const msg1 = 'Retrying with git.fetch_concurrency value of 1.'
+        logger.warn(rejections[0], msg0 + ' ' + msg1)
+        const fulfilledUrls = results.filter((it) => it?.repo.url).map((it) => it.url)
+        return collectFiles(sourcesByUrl, loadOpts, Object.assign(concurrency, { fetch: 1 }), fulfilledUrls)
+      }
+      throw rejections[0]
     }
-  } else {
-    started = tasks.map(([primary, continuation], idx) => runTask(primary, continuation, idx))
-  }
-  return Promise.all(started).then(() =>
-    Promise.all(startedContinuations).then((result) => {
-      if (rejection) throw rejection
-      return result
-    })
-  )
+    return Promise.all(
+      results.map(({ repo, authStatus, sources }) =>
+        selectStartPathsForRepository(repo, sources).then((startPaths) =>
+          collectFilesFromStartPaths.bind(null, startPaths, repo, authStatus)
+        )
+      )
+    ).then((collectTasks) => promiseAllWithLimit(collectTasks, concurrency.read))
+  })
 }
 
 function buildAggregate (componentVersionBuckets) {
-  return [
-    ...deepFlatten(componentVersionBuckets)
-      .reduce((accum, batch) => {
-        const key = batch.version + '@' + batch.name
-        const entry = accum.get(key)
-        if (!entry) return accum.set(key, batch)
+  const entries = Object.assign(new Map(), { accum: [] })
+  for (const batchesForOrigin of componentVersionBuckets) {
+    for (const batch of batchesForOrigin) {
+      let key, entry
+      if ((entry = entries.get((key = batch.version + '@' + batch.name)))) {
         const { files, origins } = batch
         ;(batch.files = entry.files).push(...files)
         ;(batch.origins = entry.origins).push(origins[0])
         Object.assign(entry, batch)
-        return accum
-      }, new Map())
-      .values(),
-  ]
+      } else {
+        entries.set(key, batch).accum.push(batch)
+      }
+    }
+  }
+  return entries.accum
 }
 
-async function loadRepository (url, opts) {
+async function loadRepository (url, opts, result = {}) {
   let authStatus, dir, repo
   const cache = { [REF_PATTERN_CACHE_KEY]: opts.refPatternCache }
   if (~url.indexOf(':') && GIT_URI_DETECTOR_RX.test(url)) {
     let credentials, displayUrl
     ;({ displayUrl, url, credentials } = extractCredentials(url))
-    const { cacheDir, fetch, fetchTags, gitPlugins, progress } = opts
+    const { cacheDir, fetch, gitPlugins, progress } = opts
     dir = ospath.join(cacheDir, generateCloneFolderName(displayUrl))
     // NOTE the presence of the url property on the repo object implies the repository is remote
     repo = { cache, dir, fs, gitdir: dir, noCheckout: true, url }
+    const { credentialManager } = gitPlugins
     const validStateFile = ospath.join(dir, VALID_STATE_FILENAME)
     try {
       await fsp.access(validStateFile)
-      if (fetch) {
+      if (fetch.always) {
         await fsp.unlink(validStateFile)
-        const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetchTags, 'fetch')
+        const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetch, 'fetch')
         await git
           .fetch(fetchOpts)
+          .then(() => ensureOids(fetchOpts))
           .then(() => {
-            authStatus = identifyAuthStatus(gitPlugins.credentialManager, credentials, url)
+            authStatus = identifyAuthStatus(credentialManager, credentials, url)
             return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
           })
           .catch((fetchErr) => {
-            if (fetchOpts.onProgress) fetchOpts.onProgress.finish(fetchErr)
+            fetchOpts.onProgress?.finish(fetchErr)
             if (HTTP_ERROR_CODE_RX.test(fetchErr.code) && fetchErr.data.statusCode === 401) fetchErr.rethrow = true
             throw fetchErr
           })
           .then(() => fsp.writeFile(validStateFile, '').catch(invariably.void))
-          .then(() => fetchOpts.onProgress && fetchOpts.onProgress.finish())
+          .then(() => fetchOpts.onProgress?.finish())
       } else {
         authStatus = await git.getConfig(Object.assign({ path: 'remote.origin.private' }, repo))
       }
     } catch (gitErr) {
       await fsp['rm' in fsp ? 'rm' : 'rmdir'](dir, { recursive: true, force: true })
       if (gitErr.rethrow) throw transformGitCloneError(gitErr, displayUrl)
-      const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetchTags, 'clone')
+      const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetch, 'clone')
       await git
         .clone(fetchOpts)
         .then(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo)))
+        .then(() => ensureOids(fetchOpts))
         .then(() => {
-          authStatus = identifyAuthStatus(gitPlugins.credentialManager, credentials, url)
+          authStatus = identifyAuthStatus(credentialManager, credentials, url)
           return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
         })
         .catch((cloneErr) => {
-          // FIXME triggering the error handler here causes assertion problems in the test suite
-          //if (fetchOpts.onProgress) fetchOpts.onProgress.finish(cloneErr)
-          throw transformGitCloneError(cloneErr, displayUrl)
+          fetchOpts.onProgress?.finish(cloneErr)
+          const authRequested = credentialManager.status({ url }) === 'requested'
+          throw transformGitCloneError(cloneErr, displayUrl, authRequested)
         })
         .then(() => fsp.writeFile(validStateFile, '').catch(invariably.void))
-        .then(() => fetchOpts.onProgress && fetchOpts.onProgress.finish())
+        .then(() => fetchOpts.onProgress?.finish())
     }
   } else if (await isDirectory((dir = expandPath(url, { dot: opts.startDir })))) {
     const dotgit = ospath.join(dir, '.git')
@@ -239,7 +233,7 @@ async function loadRepository (url, opts) {
   } else {
     throw new Error(`Local content source does not exist: ${dir}${url !== dir ? ' (url: ' + url + ')' : ''}`)
   }
-  return { repo, authStatus }
+  return Object.assign(result, { repo, authStatus })
 }
 
 function extractCredentials (url) {
@@ -254,34 +248,61 @@ function extractCredentials (url) {
     // NOTE if only username is present, assume it's an oauth token and set password to empty string
     const credentials = username ? { username, password: password || '' } : {}
     return { displayUrl, url, credentials }
-  } else if (url.startsWith('git@')) {
-    return { displayUrl: url, url: 'https://' + url.substr(4).replace(':', '/') }
-  } else {
-    return { displayUrl: url, url }
   }
+  if (url.startsWith('git@')) return { displayUrl: url, url: 'https://' + url.substr(4).replace(':', '/') }
+  return { displayUrl: url, url }
 }
 
-async function collectFilesFromSource (source, repo, remoteName, authStatus) {
-  const originUrl = repo.url || (await resolveRemoteUrl(repo, remoteName))
-  return selectReferences(source, repo, remoteName).then((refs) => {
-    if (!refs.length) {
-      const { url, branches, tags, startPath, startPaths } = source
+async function selectStartPathsForRepository (repo, sources) {
+  const startPaths = []
+  const originUrls = {}
+  for (const source of sources) {
+    const { version, editUrl } = source
+    let remoteName, originUrl
+    if (repo.url) {
+      remoteName = 'origin' // NOTE if repository is managed (has url property), we can assume remote name is origin
+      originUrl = repo.url
+    } else {
+      remoteName = source.remote || 'origin'
+      originUrl =
+        remoteName in originUrls
+          ? originUrls[remoteName]
+          : (originUrls[remoteName] = await resolveRemoteUrl(repo, remoteName))
+      if (!originUrl) {
+        remoteName = undefined
+        if ((originUrl = posixify ? 'file:///' + posixify(repo.dir) : 'file://' + repo.dir).indexOf(' ')) {
+          originUrl = originUrl.replace(SPACE_RX, '%20')
+        }
+      }
+    }
+    const refs = await selectReferences(source, repo, remoteName)
+    if (refs.length) {
+      for (const ref of refs) {
+        for (const startPath of await selectStartPaths(source, repo, ref)) {
+          startPaths.push({ startPath, ref, originUrl, editUrl, version })
+        }
+      }
+    } else {
+      const { url, branches, tags, commits } = source
       const startPathInfo =
-        'startPaths' in source ? { 'start paths': startPaths || undefined } : { 'start path': startPath || undefined }
-      const sourceInfo = yaml.dump({ url, branches, tags, ...startPathInfo }, { flowLevel: 1 }).trimRight()
+        'startPaths' in source
+          ? { 'start paths': source.startPaths || undefined }
+          : { 'start path': source.startPath || undefined }
+      const sourceInfo = yaml.dump({ url, branches, tags, commits, ...startPathInfo }, { flowLevel: 1 }).trimRight()
       logger.info(`No matching references found for content source entry (${sourceInfo.replace(NEWLINE_RX, ' | ')})`)
-      return []
     }
-    return Promise.all(refs.map((it) => collectFilesFromReference(source, repo, remoteName, authStatus, it, originUrl)))
-  })
+  }
+  return startPaths
 }
 
 // QUESTION should we resolve HEAD to a ref eagerly to avoid having to do a match on it?
 async function selectReferences (source, repo, remote) {
-  let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns } = source
-  const isBare = repo.noCheckout
+  let { branches: branchPatterns, tags: tagPatterns, commits, worktrees: worktreePatterns } = source
+  const managed = 'url' in repo
+  const isBare = managed || repo.noCheckout
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
-  const noWorktree = repo.url ? undefined : false
+  const noWorktree = managed ? undefined : false
+  const isLinkedWorktree = repo.worktree?.name
   const refs = new Map()
   if (
     tagPatterns &&
@@ -292,51 +313,77 @@ async function selectReferences (source, repo, remote) {
     const tags = await git.listTags(repo)
     if (tags.length) {
       for (const shortname of filterRefs(tags, tagPatterns, patternCache)) {
-        // NOTE tags are stored using symbol keys to distinguish them from branches
-        refs.set(Symbol(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
+        // NOTE tags are stored using Buffer keys to distinguish them from commits and branches
+        refs.set(Buffer.from(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
       }
     }
   }
-  if (!branchPatterns) return [...refs.values()]
-  branchPatterns = Array.isArray(branchPatterns)
-    ? branchPatterns.map((pattern) => String(pattern))
-    : splitRefPatterns(String(branchPatterns))
-  if (!branchPatterns.length) return [...refs.values()]
-  const worktreeName = repo.worktreeName // possibly switch to worktree property ({ name, dir}) in future
-  if (worktreeName) branchPatterns = branchPatterns.map((it) => (it === 'HEAD' ? 'HEAD@' + worktreeName : it))
-  if (worktreePatterns) {
+  if (
+    commits &&
+    (commits = Array.isArray(commits) ? commits.map((commit) => String(commit)) : commits.split(CSV_RX)).length
+  ) {
+    for (const oid of commits) {
+      // NOTE commits are stored using Symbol keys to distinguish them from tags and branches
+      refs.set(Symbol(oid), { oid, shortname: oid, fullname: 'commits/' + oid, type: 'commit' })
+    }
+  }
+  if (
+    !branchPatterns ||
+    !(branchPatterns = Array.isArray(branchPatterns)
+      ? branchPatterns.map((pattern) => String(pattern))
+      : splitRefPatterns(String(branchPatterns))).length
+  ) {
+    return [...refs.values()]
+  }
+  let useWorktree = false
+  if (!managed && (useWorktree = {})) {
     if (worktreePatterns === '.') {
-      worktreePatterns = ['.']
+      isLinkedWorktree ? (useWorktree.linked = isLinkedWorktree) : isBare || (useWorktree.main = true)
+      worktreePatterns = []
+    } else if (!worktreePatterns) {
+      worktreePatterns = []
     } else if (worktreePatterns === true) {
-      worktreePatterns = ['.', '*']
+      if (!isBare) useWorktree.main = true
+      // NOTE if we don't start at a linked worktree, linked worktree cannot be current worktree
+      if (isLinkedWorktree) useWorktree.linked = isLinkedWorktree
+      worktreePatterns = ['*']
+    } else if (worktreePatterns === '/.') {
+      if (!isBare) useWorktree.main = true
+      worktreePatterns = []
     } else {
-      worktreePatterns = Array.isArray(worktreePatterns)
-        ? worktreePatterns.map((pattern) => String(pattern))
-        : splitRefPatterns(String(worktreePatterns))
-      if (worktreeName) worktreePatterns = worktreePatterns.map((it) => (it === '@' ? worktreeName : it))
+      worktreePatterns = (
+        Array.isArray(worktreePatterns)
+          ? worktreePatterns.map((pattern) => String(pattern))
+          : splitRefPatterns(String(worktreePatterns))
+      ).reduce((accum, it) => {
+        if (it === '/.') return (isBare || (useWorktree.main = true)) && accum
+        if (it === '.') {
+          isLinkedWorktree ? (useWorktree.linked = isLinkedWorktree) : isBare || (useWorktree.main = true)
+        } else {
+          accum.push(it)
+        }
+        return accum
+      }, [])
     }
-  } else {
-    worktreePatterns = worktreePatterns === undefined ? [worktreeName || '.'] : []
+    if (!(useWorktree.main || useWorktree.linked)) useWorktree = false
   }
-  if (branchPatterns.length === 1 && (branchPatterns[0] === 'HEAD' || branchPatterns[0] === '.')) {
-    const currentBranch = await getCurrentBranchName(repo, remote)
-    if (currentBranch) {
-      branchPatterns = [currentBranch]
-    } else if (isBare) {
-      return [...refs.values()]
-    } else {
-      // NOTE current branch is undefined when HEAD is detached
-      const head = worktreePatterns[0] === '.' ? repo.dir : noWorktree
-      refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
-      return [...refs.values()]
-    }
-  } else {
+  let currentBranch
+  if (!isLinkedWorktree) {
     let headBranchIdx
-    // NOTE we can assume at least two entries if HEAD or . are present
-    if (~(headBranchIdx = branchPatterns.indexOf('HEAD')) || ~(headBranchIdx = branchPatterns.indexOf('.'))) {
-      const currentBranch = await getCurrentBranchName(repo, remote)
-      if (currentBranch) {
-        // NOTE ignore if current branch is already in list
+    if (branchPatterns.length === 1 && (branchPatterns[0] === 'HEAD' || branchPatterns[0] === '.')) {
+      if ((currentBranch = await getCurrentBranchName(repo, remote).then((branch) => branch ?? false))) {
+        branchPatterns = [currentBranch]
+      } else if (isBare) {
+        return [...refs.values()]
+      } else {
+        // NOTE current branch is undefined when HEAD is detached
+        const head = useWorktree.main ? repo.dir : noWorktree
+        refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
+        return [...refs.values()]
+      }
+    } else if (~(headBranchIdx = branchPatterns.indexOf('HEAD')) || ~(headBranchIdx = branchPatterns.indexOf('.'))) {
+      // NOTE we can assume at least two entries if HEAD or . are present
+      if ((currentBranch = await getCurrentBranchName(repo, remote).then((branch) => branch ?? false))) {
         if (~branchPatterns.indexOf(currentBranch)) {
           branchPatterns.splice(headBranchIdx, 1)
         } else {
@@ -345,11 +392,7 @@ async function selectReferences (source, repo, remote) {
       } else if (isBare) {
         branchPatterns.splice(headBranchIdx, 1)
       } else {
-        let head = noWorktree
-        if (worktreePatterns[0] === '.') {
-          worktreePatterns = worktreePatterns.slice(1)
-          head = repo.dir
-        }
+        const head = useWorktree.main ? repo.dir : noWorktree
         // NOTE current branch is undefined when HEAD is detached
         refs.set('HEAD', { shortname: 'HEAD', fullname: 'HEAD', type: 'branch', detached: true, head })
         branchPatterns.splice(headBranchIdx, 1)
@@ -357,40 +400,51 @@ async function selectReferences (source, repo, remote) {
     }
   }
   // NOTE isomorphic-git includes HEAD in list of remote branches (see https://isomorphic-git.org/docs/listBranches)
-  const remoteBranches = (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
+  const remoteBranches = remote
+    ? (await git.listBranches(Object.assign({ remote }, repo))).filter((it) => it !== 'HEAD')
+    : []
   if (remoteBranches.length) {
     for (const shortname of filterRefs(remoteBranches, branchPatterns, patternCache)) {
       const fullname = 'remotes/' + remote + '/' + shortname
       refs.set(shortname, { shortname, fullname, type: 'branch', remote, head: noWorktree })
     }
   }
-  // NOTE only consider local branches if repo has a worktree or there are no remote tracking branches
-  if (!isBare) {
-    const localBranches = await git.listBranches(repo)
-    if (localBranches.length) {
-      const worktrees = await findWorktrees(repo, worktreePatterns)
-      let onMatch
-      if ((worktreePatterns.join('') || '.') !== '.') {
-        const symbolicNames = new Map()
-        worktrees.forEach(({ name, symbolicName = 'HEAD@' + name }, shortname) => {
-          localBranches.push(symbolicName)
-          symbolicNames.set(symbolicName, shortname)
-        })
-        onMatch = (candidate, { pattern }) => {
-          const shortname = symbolicNames.get(candidate)
-          return shortname ? (pattern.startsWith('HEAD@') ? shortname : undefined) : candidate
+  if (!managed) {
+    const localBranches = await git.listBranches(repo).then((branches) => {
+      if (branches.length || isBare) return branches
+      if (currentBranch != null) return [currentBranch]
+      return getCurrentBranchName(repo).then((branch) => (branch ? [branch] : []))
+    })
+    const worktrees = await findWorktrees(repo, worktreePatterns, useWorktree)
+    let onMatch
+    if ((useWorktree || worktreePatterns.length) && worktrees.size) {
+      const headNames = new Map()
+      worktrees.forEach(({ name, symbolicNames }, shortname) => {
+        if (name) {
+          const headName = 'HEAD@' + name
+          localBranches.push(headName)
+          headNames.set(headName, shortname)
         }
-      }
-      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache, onMatch)) {
-        const head = (worktrees.get(shortname) || { head: noWorktree }).head
-        refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
+        if (symbolicNames) {
+          for (const symbolicName of symbolicNames) {
+            const symbolicHeadName = symbolicName === 'HEAD' ? symbolicName : 'HEAD@' + symbolicName
+            localBranches.push(symbolicHeadName)
+            headNames.set(symbolicHeadName, shortname)
+          }
+        }
+      })
+      onMatch = (candidate, { pattern }) => {
+        const shortname = headNames.get(candidate)
+        if (!shortname) return candidate
+        if (pattern === 'HEAD' || pattern.startsWith('HEAD@')) return shortname
       }
     }
-  } else if (!remoteBranches.length) {
-    const localBranches = await git.listBranches(repo)
     if (localBranches.length) {
-      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
-        refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head: noWorktree })
+      const preferRemote = isBare && remoteBranches.length > 0
+      for (const shortname of filterRefs(localBranches, branchPatterns, patternCache, onMatch)) {
+        if (preferRemote && refs.has(shortname)) continue
+        const head = (worktrees.get(shortname) || { head: false }).head
+        refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
       }
     }
   }
@@ -398,27 +452,28 @@ async function selectReferences (source, repo, remote) {
 }
 
 /**
- * Returns the current branch name unless the HEAD is detached.
+ * Returns the current branch name or undefined if the HEAD is detached.
  */
 function getCurrentBranchName (repo, remote) {
   return (
-    repo.noCheckout && remote
+    remote && repo.noCheckout
       ? git
-        .resolveRef(Object.assign({ ref: 'refs/remotes/' + remote + '/HEAD', depth: 2 }, repo))
-        .catch(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 2 }, repo)))
+          .resolveRef(Object.assign({ ref: 'refs/remotes/' + remote + '/HEAD', depth: 2 }, repo))
+          .catch(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 2 }, repo)))
       : git.resolveRef(Object.assign({ ref: 'HEAD', depth: 2 }, repo))
   ).then((ref) => (ref.startsWith('refs/') ? ref.replace(SHORTEN_REF_RX, '') : undefined))
 }
 
-async function collectFilesFromReference (source, repo, remoteName, authStatus, ref, originUrl) {
+async function selectStartPaths (source, repo, ref) {
   const url = repo.url
   const displayUrl = url || repo.dir
-  const { version, editUrl } = source
   const worktreePath = ref.head
   if (!worktreePath) {
-    ref.oid = await git.resolveRef(
-      Object.assign(ref.detached ? { ref: 'HEAD', depth: 1 } : { ref: 'refs/' + ref.fullname }, repo)
-    )
+    ref.oid = ref.oid
+      ? await git.expandOid(Object.assign({ oid: ref.oid }, repo)).catch(() => ref.oid)
+      : await git.resolveRef(
+          Object.assign(ref.detached ? { ref: 'HEAD', depth: 1 } : { ref: 'refs/' + ref.fullname }, repo)
+        )
   }
   if ('startPaths' in source) {
     let startPaths
@@ -433,25 +488,31 @@ async function collectFilesFromReference (source, repo, remoteName, authStatus,
       const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
       throw new Error(`no start paths found in ${where} (${ref.type}: ${ref.shortname}${flag})`)
     }
-    return Promise.all(
-      startPaths.map((startPath) =>
-        collectFilesFromStartPath(startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version)
-      )
-    )
+    return startPaths
   }
-  const startPath = cleanStartPath(coerceToString(source.startPath))
-  return collectFilesFromStartPath(startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version)
+  return [cleanStartPath(coerceToString(source.startPath))]
 }
 
-function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version) {
+async function collectFilesFromStartPaths (startPaths, repo, authStatus) {
+  const buckets = []
+  for (const { startPath, ref, originUrl, editUrl, version } of startPaths) {
+    buckets.push(await collectFilesFromStartPath(startPath, repo, authStatus, ref, originUrl, editUrl, version))
+  }
+  repo.cache = undefined
+  return buckets
+}
+
+function collectFilesFromStartPath (startPath, repo, authStatus, ref, originUrl, editUrl, version) {
+  const worktreePath = ref.head
   const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
   return (worktreePath ? readFilesFromWorktree(origin) : readFilesFromGitTree(repo, ref.oid, startPath))
-    .then((files) =>
-      Object.assign(deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version))), {
-        files: files.map((file) => assignFileProperties(file, origin)),
-        origins: [origin],
-      })
-    )
+    .then((files) => {
+      const batch = deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version)))
+      if ('nav' in batch && Array.isArray(batch.nav)) batch.nav.origin = origin
+      batch.files = files.map((file) => assignFileProperties(file, origin))
+      batch.origins = [origin]
+      return batch
+    })
     .catch((err) => {
       const where = worktreePath || (worktreePath === false ? repo.gitdir : repo.url || repo.dir)
       const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
@@ -476,19 +537,18 @@ function readFilesFromWorktree (origin) {
 }
 
 function srcFs (cwd, origin) {
-  return new Promise((resolve, reject, cache = Object.create(null), files = [], relpathStart = cwd.length + 1) =>
+  return new Promise((resolve, reject, files = []) =>
     pipeline(
-      globStream(CONTENT_SRC_GLOB, Object.assign({ cache, cwd }, CONTENT_SRC_OPTS)),
-      forEach(({ path: abspathPosix }, _, done) => {
-        if ((cache[abspathPosix] || {}).constructor === Array) return done() // detects some directories
-        const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
-        const relpath = abspath.substr(relpathStart)
-        symlinkAwareStat(abspath).then(
-          (stat) => {
-            if (stat.isDirectory()) return done() // detects directories that slipped through cache check
+      globStream(CONTENT_SRC_GLOB, Object.assign({ cwd }, CONTENT_SRC_OPTS)),
+      forEach(({ path: relpath, dirent }, _, done) => {
+        if (dirent.isDirectory()) return done()
+        const relpathPosix = relpath
+        const abspath = posixify ? ospath.join(cwd, (relpath = ospath.normalize(relpath))) : cwd + '/' + relpath
+        fsp.stat(abspath).then(
+          (stat) =>
             fsp.readFile(abspath).then(
               (contents) => {
-                files.push(new File({ path: posixify ? posixify(relpath) : relpath, contents, stat, src: { abspath } }))
+                files.push(new File({ path: relpathPosix, contents, stat, src: { abspath } }))
                 done()
               },
               (readErr) => {
@@ -498,22 +558,28 @@ function srcFs (cwd, origin) {
                   : logger.error(logObject, readErr.message.replace(`'${abspath}'`, relpath))
                 done()
               }
-            )
-          },
+            ),
           (statErr) => {
             const logObject = { file: { abspath, origin } }
-            if (statErr.symlink) {
-              logger.error(
-                logObject,
-                (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
-                  `${relpath} -> ${statErr.symlink}`
-              )
-            } else if (statErr.code === 'ENOENT') {
-              logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
+            if (dirent.isSymbolicLink()) {
+              fsp
+                .readlink(abspath)
+                .then(
+                  (symlink) =>
+                    (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
+                    `${relpath} -> ${symlink}`,
+                  () => statErr.message.replace(`'${abspath}'`, relpath)
+                )
+                .then((message) => {
+                  logger.error(logObject, message)
+                  done()
+                })
             } else {
-              logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
+              statErr.code === 'ENOENT'
+                ? logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
+                : logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
+              done()
             }
-            done()
           }
         )
       }),
@@ -523,26 +589,25 @@ function srcFs (cwd, origin) {
 }
 
 function readFilesFromGitTree (repo, oid, startPath) {
-  return git
-    .readTree(Object.assign({ oid }, repo))
-    .then((root) =>
-      getGitTreeAtStartPath(repo, oid, startPath).then((start) =>
-        srcGitTree(repo, Object.assign(root, { dirname: '' }), start)
-      )
-    )
+  return git.readTree(Object.assign({ oid }, repo)).then((root) => {
+    Object.assign(root, { dirname: '' })
+    return startPath
+      ? getGitTreeAtStartPath(repo, oid, startPath).then((start) => {
+          Object.assign(start, { dirname: startPath })
+          return srcGitTree(repo, root, start)
+        })
+      : srcGitTree(repo, root)
+  })
 }
 
 function getGitTreeAtStartPath (repo, oid, startPath) {
-  return git.readTree(Object.assign({ oid, filepath: startPath }, repo)).then(
-    (result) => Object.assign(result, { dirname: startPath }),
-    (err) => {
-      const m = err instanceof ObjectTypeError && err.data.expected === 'tree' ? 'is not a directory' : 'does not exist'
-      throw new Error(`the start path '${startPath}' ${m}`)
-    }
-  )
+  return git.readTree(Object.assign({ oid, filepath: startPath }, repo)).catch((err) => {
+    const m = err instanceof ObjectTypeError && err.data.expected === 'tree' ? 'is not a directory' : 'does not exist'
+    throw new Error(`the start path '${startPath}' ${m}`)
+  })
 }
 
-function srcGitTree (repo, root, start) {
+function srcGitTree (repo, root, start = root) {
   return new Promise((resolve, reject) => {
     const files = []
     createGitTreeWalker(repo, root, filterGitEntry, gitEntryToFile)
@@ -585,7 +650,8 @@ function visitGitTree (emitter, repo, root, filter, convert, parent, dirname = '
               (target) => {
                 if (target.type === 'tree') {
                   return visitGitTree(emitter, repo, root, filter, convert, target, vfilePath, target.following)
-                } else if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
+                }
+                if (target.type === 'blob' && filterVerdict === true && (mode = FILE_MODES[target.mode])) {
                   return convert(Object.assign({ mode, oid: target.oid, path: vfilePath }, repo)).then((result) =>
                     emitter.emit('entry', result)
                   )
@@ -657,11 +723,11 @@ function readGitObjectAtPath (repo, root, parent, pathSegments, following) {
     if (entry.path === firstPathSegment) {
       return entry.type === 'tree'
         ? git.readTree(Object.assign({ oid: entry.oid }, repo)).then((subtree) => {
-          Object.assign(subtree, { dirname: path.join(parent.dirname, entry.path) })
-          return (pathSegments = pathSegments.slice(1)).length
-            ? readGitObjectAtPath(repo, root, subtree, pathSegments, following)
-            : Object.assign(subtree, { type: 'tree', following }) // Q: should this create copy?
-        })
+            Object.assign(subtree, { dirname: path.join(parent.dirname, entry.path) })
+            return (pathSegments = pathSegments.slice(1)).length
+              ? readGitObjectAtPath(repo, root, subtree, pathSegments, following)
+              : Object.assign(subtree, { type: 'tree', following }) // Q: should this create copy?
+          })
         : entry.mode === SYMLINK_FILE_MODE
           ? readGitSymlink(repo, root, parent, entry, following)
           : Promise.resolve(entry)
@@ -684,8 +750,13 @@ function filterGitEntry (entry) {
 
 function gitEntryToFile (entry) {
   return git.readBlob(entry).then(({ blob: contents }) => {
-    contents = Buffer.from(contents.buffer)
-    const stat = Object.assign(new fs.Stats(), { mode: entry.mode, mtime: undefined, size: contents.byteLength })
+    const stat = {
+      mode: entry.mode,
+      size: (contents = Buffer.from(contents.buffer)).byteLength,
+      isDirectory: invariably.false,
+      isFile: invariably.true,
+      isSymbolicLink: invariably.false,
+    }
     return new File({ path: entry.path, contents, stat })
   })
 }
@@ -697,7 +768,7 @@ function loadComponentDescriptor (files, ref, version) {
   files.splice(descriptorFileIdx, 1)
   let data
   try {
-    data = yaml.load(descriptorFile.contents.toString(), { schema: yaml.CORE_SCHEMA })
+    data = Object(yaml.load(descriptorFile.contents.toString(), { schema: yaml.CORE_SCHEMA }))
   } catch (err) {
     throw Object.assign(err, { message: `${COMPONENT_DESC_FILENAME} has invalid syntax; ${err.message}` })
   }
@@ -710,18 +781,18 @@ function loadComponentDescriptor (files, ref, version) {
   if (!version) {
     if (version === undefined) throw new Error(`${COMPONENT_DESC_FILENAME} is missing a version`)
     if (version === false) throw new Error(`${COMPONENT_DESC_FILENAME} has an invalid version`)
-    version = '' + (typeof version === 'number' ? version : '')
+    version = typeof version === 'number' ? '' + version : ''
   } else if (version === true) {
     version = ref.shortname.replace(PATH_SEPARATOR_RX, '-')
   } else if (version.constructor === Object) {
     const refname = ref.shortname
     let matched
     if (refname in version) {
-      matched = version[refname]
+      matched = '' + (version[refname] ?? '')
     } else if (
       !Object.entries(version).some(([pattern, replacement]) => {
-        const result = refname.replace(makeMatcherRx(pattern, VERSION_MATCHER_OPTS), '\0' + replacement)
-        if (result === refname) return false
+        const result = refname.replace(makeMatcherRx(pattern, VERSION_MATCHER_OPTS), '\0' + (replacement ?? ''))
+        if (result === refname) return false // no match
         matched = result.substr(1)
         return true
       })
@@ -736,7 +807,7 @@ function loadComponentDescriptor (files, ref, version) {
     throw new Error(`version in ${COMPONENT_DESC_FILENAME} cannot have path segments: ${version}`)
   }
   data.version = version
-  return camelCaseKeys(data, ['asciidoc'])
+  return camelCaseKeys(data, ['asciidoc', 'ext'])
 }
 
 function assignFileProperties (file, origin) {
@@ -753,20 +824,23 @@ function assignFileProperties (file, origin) {
   return file
 }
 
-function buildFetchOptions (repo, progress, displayUrl, credentialsFromUrl, gitPlugins, fetchTags, operation) {
+function buildFetchOptions (repo, progress, displayUrl, credentialsFromUrl, gitPlugins, fetch, operation) {
   const { credentialManager, http, urlRouter } = gitPlugins
+  const corsProxy = false
+  const depth = fetch.depth || undefined
   const onAuth = resolveCredentials.bind(credentialManager, new Map().set(undefined, credentialsFromUrl))
   const onAuthFailure = onAuth
   const onAuthSuccess = (url) => credentialManager.approved({ url })
-  const opts = Object.assign({ corsProxy: false, depth: 1, http, onAuth, onAuthFailure, onAuthSuccess }, repo)
+  const opts = Object.assign({ corsProxy, depth, http, onAuth, onAuthFailure, onAuthSuccess }, repo)
   if (urlRouter) opts.url = urlRouter.ensureGitSuffix(opts.url)
   if (progress) opts.onProgress = createProgressListener(progress, displayUrl, operation)
   if (operation === 'fetch') {
     opts.prune = true
-    if (fetchTags) opts.tags = opts.pruneTags = true
-  } else if (!fetchTags) {
+    if (fetch.tags) opts.tags = opts.pruneTags = true
+  } else if (!fetch.tags) {
     opts.noTags = true
   }
+  if (fetch.commits) opts.oids = fetch.commits
   return opts
 }
 
@@ -801,7 +875,10 @@ function createProgressListener (progress, progressLabel, operation) {
   // NOTE leave room for indeterminate progress at end of bar; this isn't strictly needed for a bare clone
   progressBar.scaleFactor = Math.max(0, (ticks - 1) / ticks)
   progressBar.tick(0)
-  return Object.assign(onGitProgress.bind(progressBar), { finish: onGitComplete.bind(progressBar) })
+  return Object.assign(onGitProgress.bind(progressBar), {
+    finish: onGitComplete.bind(progressBar),
+    reset: () => progressBar.update(0),
+  })
 }
 
 function formatProgressBar (label, maxLabelWidth, operation) {
@@ -829,7 +906,6 @@ function onGitProgress ({ phase, loaded, total }) {
 
 function onGitComplete (err) {
   if (err) {
-    // TODO could use progressBar.interrupt() to replace bar with message instead
     this.chars.incomplete = '?'
     this.update(0)
     // NOTE force progress bar to update regardless of throttle setting
@@ -863,9 +939,8 @@ function identifyAuthStatus (credentialManager, credentials, url) {
   const status = credentialManager.status({ url })
   if (credentials) {
     return typeof status === 'string' && status.startsWith('requested,') ? 'auth-required' : 'auth-embedded'
-  } else if (status != null) {
-    return 'auth-required'
   }
+  if (status != null) return 'auth-required'
 }
 
 /**
@@ -874,7 +949,7 @@ function identifyAuthStatus (credentialManager, credentials, url) {
  * The purpose of this function is generate a safe, unique folder name for the cloned
  * repository that gets stored in the cache directory.
  *
- * The generated folder name follows the pattern: <basename>-<sha1>-<version>.git
+ * The generated folder name follows the pattern: <basename>-<sha1-of-normalized-url>.git
  *
  * @param {String} url - The repository URL to convert.
  * @returns {String} The generated folder name.
@@ -890,21 +965,18 @@ function generateCloneFolderName (url) {
  *
  * @param {Repository} repo - The repository on which to operate.
  * @param {String} remoteName - The name of the remote to resolve.
- * @returns {String} The URL of the specified remote, if defined, or the file URI to the local repository.
+ * @returns {String} The URL of the specified remote, if defined
  */
 function resolveRemoteUrl (repo, remoteName) {
   return git.getConfig(Object.assign({ path: 'remote.' + remoteName + '.url' }, repo)).then((url) => {
-    if (url) {
-      if (url.startsWith('https://') || url.startsWith('http://')) {
-        return ~url.indexOf('@') ? url.replace(URL_AUTH_CLEANER_RX, '$1') : url
-      } else if (url.startsWith('git@')) {
-        return 'https://' + url.substr(4).replace(':', '/')
-      } else if (url.startsWith('ssh://')) {
-        return 'https://' + url.substr(url.indexOf('@') + 1 || 6).replace(URL_PORT_CLEANER_RX, '$1')
-      }
+    if (!url) return
+    if (url.startsWith('https://') || url.startsWith('http://')) {
+      return ~url.indexOf('@') ? url.replace(URL_AUTH_CLEANER_RX, '$1') : url
+    }
+    if (url.startsWith('git@')) return 'https://' + url.substr(4).replace(':', '/')
+    if (url.startsWith('ssh://')) {
+      return 'https://' + url.substr(url.indexOf('@') + 1 || 6).replace(URL_PORT_CLEANER_RX, '$1')
     }
-    url = posixify ? 'file:///' + posixify(repo.dir) : 'file://' + repo.dir
-    return ~url.indexOf(' ') ? url.replace(SPACE_RX, '%20') : url
   })
 }
 
@@ -918,24 +990,20 @@ function isDirectory (url) {
   return fsp.stat(url).then((stat) => stat.isDirectory(), invariably.false)
 }
 
-function symlinkAwareStat (path_) {
-  return fsp.lstat(path_).then((lstat) => {
-    if (!lstat.isSymbolicLink()) return lstat
-    return fsp.stat(path_).catch((statErr) =>
-      fsp
-        .readlink(path_)
-        .catch(invariably.void)
-        .then((symlink) => {
-          throw Object.assign(statErr, { symlink })
-        })
-    )
-  })
-}
-
 function tagsSpecified (sources) {
   return sources.some(({ tags }) => tags && (Array.isArray(tags) ? tags.length : true))
 }
 
+function commitsRequested (sources) {
+  if (!sources.some(({ commits }) => commits && (Array.isArray(commits) ? commits.length : true))) return
+  const result = new Set()
+  for (const { commits } of sources) {
+    if (!commits) continue
+    for (const commit of Array.isArray(commits) ? commits : commits.split(CSV_RX)) result.add(String(commit))
+  }
+  return [...result]
+}
+
 function loadGitPlugins (gitConfig, networkConfig, startDir) {
   const plugins = new Map((git.cores || git.default.cores || new Map()).get(GIT_CORE))
   for (const [name, request] of Object.entries(gitConfig.plugins || {})) {
@@ -946,7 +1014,7 @@ function loadGitPlugins (gitConfig, networkConfig, startDir) {
     if (typeof credentialManager.configure === 'function') {
       credentialManager.configure({ config: gitConfig.credentials, startDir })
     }
-    if (typeof credentialManager.status !== 'function') Object.assign(credentialManager, { status () {} })
+    if (typeof credentialManager.status !== 'function') Object.assign(credentialManager, { status: invariably.void })
   } else {
     credentialManager = new GitCredentialManagerStore().configure({ config: gitConfig.credentials, startDir })
   }
@@ -979,8 +1047,8 @@ function ensureCacheDir (preferredCacheDir, startDir) {
   )
 }
 
-function transformGitCloneError (err, displayUrl) {
-  let wrappedMsg, trimMessage
+function transformGitCloneError (err, displayUrl, authRequested) {
+  let wrappedMsg, recoverable, trimMessage
   if (HTTP_ERROR_CODE_RX.test(err.code)) {
     switch (err.data.statusCode) {
       case 401:
@@ -989,30 +1057,31 @@ function transformGitCloneError (err, displayUrl) {
           : 'Content repository not found or requires credentials'
         break
       case 404:
-        wrappedMsg = 'Content repository not found'
+        wrappedMsg = authRequested
+          ? 'Content repository not found or credentials were rejected'
+          : 'Content repository not found'
         break
       default:
         wrappedMsg = err.message
-        trimMessage = true
+        recoverable = trimMessage = true
     }
   } else if (err instanceof UrlParseError || err instanceof UnknownTransportError) {
     wrappedMsg = 'Content source uses an unsupported transport protocol'
   } else if (err.code === 'ENOTFOUND') {
     wrappedMsg = `Content repository host could not be resolved: ${err.hostname}`
   } else {
-    wrappedMsg = `${err.name}: ${err.message}`
-    trimMessage = true
-  }
-  if (trimMessage) {
-    wrappedMsg = ~(wrappedMsg = wrappedMsg.trimRight()).indexOf('. ') ? wrappedMsg : wrappedMsg.replace(/\.$/, '')
+    wrappedMsg = err.message || String(err)
+    recoverable = trimMessage = true
   }
+  if (trimMessage && !~(wrappedMsg = wrappedMsg.trimEnd()).indexOf('. ')) wrappedMsg = wrappedMsg.replace(/\.$/, '')
   const errWrapper = new Error(`${wrappedMsg} (url: ${displayUrl})`)
-  errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
+  errWrapper.stack += `\nCaused by: ${err.stack ? inspect(err).replace(/^Error \[(.+?)\](?=: )/, '$1') : err}`
+  if (recoverable) Object.defineProperty(errWrapper, 'recoverable', { value: true })
   return errWrapper
 }
 
 function splitRefPatterns (str) {
-  return ~str.indexOf('{') ? str.split(VENTILATED_CSV_RX) : str.split(CSV_RX)
+  return str.split(~str.indexOf('{') ? VENTILATED_CSV_RX : CSV_RX)
 }
 
 function camelCaseKeys (o, stopPaths = [], p = '') {
@@ -1035,54 +1104,116 @@ function coerceToString (value) {
   return value == null ? '' : String(value)
 }
 
-async function resolveRepositoryFromWorktree (repo) {
+function resolveRepositoryFromWorktree (repo) {
   return fsp
     .readFile(repo.gitdir, 'utf8')
-    .then((contents) => contents.trimRight().substr(8))
-    .then((worktreeGitdir) => {
-      const worktreeName = ospath.basename(worktreeGitdir)
-      return fsp.readFile(ospath.join(worktreeGitdir, 'commondir'), 'utf8').then(
+    .then((contents) => contents.substr(8).trimEnd())
+    .then((worktreeGitdir) =>
+      fsp.readFile(ospath.join(worktreeGitdir, 'commondir'), 'utf8').then(
         (contents) => {
-          const gitdir = ospath.join(worktreeGitdir, contents.trimRight())
-          return ospath.basename(gitdir) === '.git'
-            ? Object.assign(repo, { dir: ospath.dirname(gitdir), gitdir, worktreeName })
-            : Object.assign(repo, { dir: gitdir, gitdir, worktreeName })
+          const gitdir = ospath.join(worktreeGitdir, contents.trimEnd())
+          const dir = ospath.basename(gitdir) === '.git' ? ospath.dirname(gitdir) : gitdir
+          const name = ospath.basename(worktreeGitdir)
+          return Object.assign(repo, { dir, gitdir, worktree: { gitdir: worktreeGitdir, name } })
         },
         () => repo
       )
-    })
+    )
 }
 
-function findWorktrees (repo, patterns) {
-  if (!patterns.length) return new Map()
-  const mainWorktree =
-    patterns[0] === '.' && (patterns = patterns.slice(1))
-      ? getCurrentBranchName(repo).then((branch) => (branch ? [branch, { head: repo.dir, name: '.' }] : undefined))
-      : Promise.resolve()
-  const worktreesDir = patterns.length ? ospath.join(repo.dir, '.git', 'worktrees') : undefined
+function findWorktrees (repo, patterns, useWorktree) {
+  const useLinkedWorktree = !!useWorktree.linked
+  const mainWorktree = useWorktree.main
+    ? getCurrentBranchName(repo).then((branch) => {
+        if (!branch) return
+        return [branch, { head: repo.dir, name: undefined, symbolicNames: useLinkedWorktree ? ['/.'] : ['/.', '.'] }]
+      })
+    : Promise.resolve()
+  if (!(useLinkedWorktree || patterns.length)) return mainWorktree.then((entry) => new Map(entry && [entry]))
+  const worktreesDir = ospath.join(repo.dir, repo.dir === repo.gitdir ? '' : '.git', 'worktrees')
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
-  return (
-    worktreesDir
-      ? fsp
+  const scanWorktrees = patterns.length
+    ? fsp
         .readdir(worktreesDir)
         .then((worktreeNames) => filterRefs(worktreeNames, patterns, patternCache), invariably.emptyArray)
-        .then((worktreeNames) =>
-          Promise.all(
-            worktreeNames.map((worktreeName) => {
-              const gitdir = ospath.resolve(worktreesDir, worktreeName)
-              // NOTE branch name defaults to worktree name if HEAD is detached
-              return git
-                .currentBranch(Object.assign({}, repo, { gitdir }))
-                .then((branch = worktreeName) =>
-                  fsp
-                    .readFile(ospath.join(gitdir, 'gitdir'), 'utf8')
-                    .then((contents) => [branch, { head: ospath.dirname(contents.trimRight()), name: worktreeName }])
-                )
-            })
+        .then((worktreeNames) => {
+          if (useLinkedWorktree && !~worktreeNames.indexOf(useWorktree.linked)) worktreeNames.push(useWorktree.linked)
+          return worktreeNames
+        })
+    : Promise.resolve(useLinkedWorktree ? [useWorktree.linked] : [])
+  return scanWorktrees
+    .then((worktreeNames) =>
+      Promise.all(
+        worktreeNames.map((name) => {
+          const symbolicNames = useLinkedWorktree && name === useWorktree.linked ? ['.', 'HEAD'] : undefined
+          const gitdir = ospath.resolve(worktreesDir, name)
+          // NOTE branch name defaults to worktree name if HEAD is detached
+          return getCurrentBranchName(Object.assign({}, repo, { gitdir })).then((branch = name) =>
+            fsp
+              .readFile(ospath.join(gitdir, 'gitdir'), 'utf8')
+              .then((contents) => [branch, { head: ospath.dirname(contents.trimEnd()), name, symbolicNames }])
           )
-        )
-      : Promise.resolve()
-  ).then((entries = []) => mainWorktree.then((entry) => new Map(entry ? entries.push(entry) && entries : entries)))
+        })
+      )
+    )
+    .then((entries) => new Map(entries))
+    .then((worktrees) => mainWorktree.then((result) => (result ? worktrees.set(result[0], result[1]) : worktrees)))
+}
+
+async function gracefulPromiseAllWithLimit (tasks, limit = Infinity) {
+  const rejections = []
+  const recordRejection = (err) => rejections.push(err) && undefined
+  const started = []
+  if (tasks.length <= limit) {
+    for (const task of tasks) started.push(task().catch(recordRejection))
+  } else {
+    const pending = []
+    for (const task of tasks) {
+      const current = task()
+        .catch(recordRejection)
+        .finally(() => pending.splice(pending.indexOf(current), 1))
+      started.push(current)
+      if (pending.push(current) < limit) continue
+      await Promise.race(pending)
+      if (rejections.length) break
+    }
+  }
+  return Promise.all(started).then((results) => [results, rejections])
+}
+
+async function promiseAllWithLimit (tasks, limit = Infinity) {
+  if (tasks.length <= limit) return Promise.all(tasks.map((task) => task()))
+  const started = []
+  const pending = []
+  for (const task of tasks) {
+    const current = task().finally(() => pending.splice(pending.indexOf(current), 1))
+    started.push(current)
+    if (pending.push(current) >= limit) await Promise.race(pending)
+  }
+  return Promise.all(started)
+}
+
+async function ensureOids (opts) {
+  if (!opts.oids) return
+  let prevShallowCommits = await getShallowCommits(opts)
+  if (prevShallowCommits == null) return
+  let oids = opts.oids.slice()
+  const deepenOpts = Object.assign({}, opts, { relative: true })
+  const format = 'deflated'
+  while (oids.length) {
+    deepenOpts.onProgress?.reset()
+    await git.fetch(deepenOpts)
+    const shallowCommits = await getShallowCommits(opts)
+    if (shallowCommits == null || shallowCommits === prevShallowCommits) break
+    prevShallowCommits = shallowCommits
+    oids = await Promise.all(
+      oids.map((oid) => git.readObject(Object.assign({ oid, format }, opts)).then(invariably.void, () => oid))
+    ).then((results) => results.filter((it) => it))
+  }
+}
+
+function getShallowCommits ({ gitdir }) {
+  return fsp.readFile(ospath.join(gitdir, 'shallow'), 'utf8').catch(invariably.void)
 }
 
 module.exports = aggregateContent