@antora/content-aggregator 3.2.0-alpha.2 → 3.2.0-alpha.4

package/README.md

@@ -9,4 +9,4 @@ Its site generator aggregates documents from versioned content repositories and
 
 Copyright (C) 2017-present [OpenDevise Inc.](https://opendevise.com) and the [Antora Project](https://antora.org).
 
-Use of this software is granted under the terms of the [Mozilla Public License Version 2.0](https://www.mozilla.org/en-US/MPL/2.0/) (MPL-2.0).
+Use of this software is granted under the terms of the [Mozilla Public License Version 2.0](https://www.mozilla.org/en-US/MPL/2.0/) (MPL-2.0).

package/lib/aggregate-content.js

@@ -5,7 +5,6 @@ const { createHash } = require('crypto')
 const createGitHttpPlugin = require('./git-plugin-http')
 const decodeUint8Array = require('./decode-uint8-array')
 const deepClone = require('./deep-clone')
-const deepFlatten = require('./deep-flatten')
 const EventEmitter = require('events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
@@ -17,6 +16,7 @@ const GitCredentialManagerStore = require('./git-credential-manager-store')
 const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
 const globStream = require('glob-stream')
+const { inspect } = require('util')
 const invariably = require('./invariably')
 const logger = require('./logger')
 const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')
@@ -93,83 +93,71 @@ function aggregateContent (playbook) {
   return ensureCacheDir(requestedCacheDir, startDir).then((cacheDir) => {
     const gitConfig = Object.assign({ ensureGitSuffix: true }, playbook.git)
     const gitPlugins = loadGitPlugins(gitConfig, playbook.network || {}, startDir)
-    const fetchConcurrency = Math.max(gitConfig.fetchConcurrency || Infinity, 1)
+    const concurrency = {
+      fetch: Math.max(gitConfig.fetchConcurrency || Infinity, 1),
+      read: Math.max(gitConfig.readConcurrency || Infinity, 1),
+    }
     const sourcesByUrl = sources.reduce((accum, source) => {
       return accum.set(source.url, [...(accum.get(source.url) || []), Object.assign({}, sourceDefaults, source)])
     }, new Map())
     const progress = !quiet && createProgress(sourcesByUrl.keys(), process.stdout)
     const refPatternCache = Object.assign(new Map(), { braces: new Map() })
     const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir, refPatternCache }
-    return collectFiles(sourcesByUrl, loadOpts, fetchConcurrency).then(buildAggregate, (err) => {
+    return collectFiles(sourcesByUrl, loadOpts, concurrency).then(buildAggregate, (err) => {
       progress && progress.terminate()
       throw err
     })
   })
 }
 
-async function collectFiles (sourcesByUrl, loadOpts, concurrency) {
-  const tasks = [...sourcesByUrl.entries()].map(([url, sources]) => [
-    () => loadRepository(url, Object.assign({ fetchTags: tagsSpecified(sources) }, loadOpts)),
-    ({ repo, authStatus }) =>
-      Promise.all(
-        sources.map((source) => {
-          // NOTE if repository is managed (has a url property), we can assume the remote name is origin
-          // TODO if the repo has no remotes, then remoteName should be undefined
-          const remoteName = repo.url ? 'origin' : source.remote || 'origin'
-          return collectFilesFromSource(source, repo, remoteName, authStatus)
-        })
-      ),
-  ])
-  let rejection, started
-  const startedContinuations = []
-  const recordRejection = (err) => {
-    rejection = err
-  }
-  const runTask = (primary, continuation, idx) =>
-    primary().then((value) => {
-      if (!rejection) startedContinuations[idx] = continuation(value).catch(recordRejection)
-    }, recordRejection)
-  if (tasks.length > concurrency) {
-    started = []
-    const pending = []
-    for (const [primary, continuation] of tasks) {
-      const current = runTask(primary, continuation, started.length).finally(() =>
-        pending.splice(pending.indexOf(current), 1)
-      )
-      started.push(current)
-      if (pending.push(current) < concurrency) continue
-      await Promise.race(pending)
-      if (rejection) break
+async function collectFiles (sourcesByUrl, loadOpts, concurrency, fetchedUrls) {
+  const loadTasks = [...sourcesByUrl.entries()].map(([url, sources]) => {
+    const loadOptsForUrl = Object.assign({}, loadOpts)
+    if (loadOpts.fetch && fetchedUrls && fetchedUrls.length && fetchedUrls.includes(url)) loadOptsForUrl.fetch = false
+    if (tagsSpecified(sources)) loadOptsForUrl.fetchTags = true
+    return loadRepository.bind(null, url, loadOptsForUrl, { url, sources })
+  })
+  return gracefulPromiseAllWithLimit(loadTasks, concurrency.fetch).then(([results, rejections]) => {
+    if (rejections.length) {
+      if (concurrency.fetch > 1 && results.length > 1 && rejections.every(({ recoverable }) => recoverable)) {
+        if (loadOpts.progress) loadOpts.progress.terminate() // reset cursor position and allow it be reused
+        const msg0 = 'An unexpected error occurred while fetching content sources concurrently.'
+        const msg1 = 'Retrying with git.fetch_concurrency value of 1.'
+        logger.warn(rejections[0], msg0 + ' ' + msg1)
+        const fulfilledUrls = results.map((it) => it && it.repo.url && it.url).filter((it) => it)
+        return collectFiles(sourcesByUrl, loadOpts, Object.assign(concurrency, { fetch: 1 }), fulfilledUrls)
+      }
+      throw rejections[0]
     }
-  } else {
-    started = tasks.map(([primary, continuation], idx) => runTask(primary, continuation, idx))
-  }
-  return Promise.all(started).then(() =>
-    Promise.all(startedContinuations).then((result) => {
-      if (rejection) throw rejection
-      return result
-    })
-  )
+    return Promise.all(
+      results.map(({ repo, authStatus, sources }) =>
+        selectStartPathsForRepository(repo, authStatus, sources).then((startPaths) =>
+          collectFilesFromStartPaths.bind(null, startPaths, repo, authStatus)
+        )
+      )
+    ).then((collectTasks) => promiseAllWithLimit(collectTasks, concurrency.read))
+  })
 }
 
 function buildAggregate (componentVersionBuckets) {
-  return [
-    ...deepFlatten(componentVersionBuckets)
-      .reduce((accum, batch) => {
-        const key = batch.version + '@' + batch.name
-        const entry = accum.get(key)
-        if (!entry) return accum.set(key, batch)
+  const entries = Object.assign(new Map(), { accum: [] })
+  for (const batchesForOrigin of componentVersionBuckets) {
+    for (const batch of batchesForOrigin) {
+      let key, entry
+      if ((entry = entries.get((key = batch.version + '@' + batch.name)))) {
         const { files, origins } = batch
         ;(batch.files = entry.files).push(...files)
         ;(batch.origins = entry.origins).push(origins[0])
         Object.assign(entry, batch)
-        return accum
-      }, new Map())
-      .values(),
-  ]
+      } else {
+        entries.set(key, batch).accum.push(batch)
+      }
+    }
+  }
+  return entries.accum
 }
 
-async function loadRepository (url, opts) {
+async function loadRepository (url, opts, result = {}) {
   let authStatus, dir, repo
   const cache = { [REF_PATTERN_CACHE_KEY]: opts.refPatternCache }
   if (~url.indexOf(':') && GIT_URI_DETECTOR_RX.test(url)) {
@@ -179,6 +167,7 @@ async function loadRepository (url, opts) {
     dir = ospath.join(cacheDir, generateCloneFolderName(displayUrl))
     // NOTE the presence of the url property on the repo object implies the repository is remote
     repo = { cache, dir, fs, gitdir: dir, noCheckout: true, url }
+    const { credentialManager } = gitPlugins
     const validStateFile = ospath.join(dir, VALID_STATE_FILENAME)
     try {
       await fsp.access(validStateFile)
@@ -188,7 +177,7 @@ async function loadRepository (url, opts) {
         await git
           .fetch(fetchOpts)
           .then(() => {
-            authStatus = identifyAuthStatus(gitPlugins.credentialManager, credentials, url)
+            authStatus = identifyAuthStatus(credentialManager, credentials, url)
             return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
           })
           .catch((fetchErr) => {
@@ -209,13 +198,13 @@ async function loadRepository (url, opts) {
         .clone(fetchOpts)
         .then(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo)))
         .then(() => {
-          authStatus = identifyAuthStatus(gitPlugins.credentialManager, credentials, url)
+          authStatus = identifyAuthStatus(credentialManager, credentials, url)
           return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
         })
         .catch((cloneErr) => {
-          // FIXME triggering the error handler here causes assertion problems in the test suite
-          //if (fetchOpts.onProgress) fetchOpts.onProgress.finish(cloneErr)
-          throw transformGitCloneError(cloneErr, displayUrl)
+          if (fetchOpts.onProgress) fetchOpts.onProgress.finish(cloneErr)
+          const authRequested = credentialManager.status({ url }) === 'requested'
+          throw transformGitCloneError(cloneErr, displayUrl, authRequested)
         })
         .then(() => fsp.writeFile(validStateFile, '').catch(invariably.void))
         .then(() => fetchOpts.onProgress && fetchOpts.onProgress.finish())
@@ -239,7 +228,7 @@ async function loadRepository (url, opts) {
   } else {
     throw new Error(`Local content source does not exist: ${dir}${url !== dir ? ' (url: ' + url + ')' : ''}`)
   }
-  return { repo, authStatus }
+  return Object.assign(result, { repo, authStatus })
 }
 
 function extractCredentials (url) {
@@ -261,27 +250,42 @@ function extractCredentials (url) {
   }
 }
 
-async function collectFilesFromSource (source, repo, remoteName, authStatus) {
-  const originUrl = repo.url || (await resolveRemoteUrl(repo, remoteName))
-  return selectReferences(source, repo, remoteName).then((refs) => {
-    if (!refs.length) {
-      const { url, branches, tags, startPath, startPaths } = source
+async function selectStartPathsForRepository (repo, authStatus, sources) {
+  const startPaths = []
+  const originUrls = {}
+  for (const source of sources) {
+    const { version, editUrl } = source
+    // NOTE if repository is managed (has a url property), we can assume the remote name is origin
+    // TODO if the repo has no remotes, then remoteName should be undefined
+    const remoteName = repo.url ? 'origin' : source.remote || 'origin'
+    const originUrl = repo.url || (originUrls[remoteName] ||= await resolveRemoteUrl(repo, remoteName))
+    const refs = await selectReferences(source, repo, remoteName)
+    if (refs.length) {
+      for (const ref of refs) {
+        for (const startPath of await selectStartPaths(source, repo, remoteName, ref)) {
+          startPaths.push({ startPath, ref, originUrl, editUrl, version })
+        }
+      }
+    } else {
+      const { url, branches, tags } = source
       const startPathInfo =
-        'startPaths' in source ? { 'start paths': startPaths || undefined } : { 'start path': startPath || undefined }
+        'startPaths' in source
+          ? { 'start paths': source.startPaths || undefined }
+          : { 'start path': source.startPath || undefined }
       const sourceInfo = yaml.dump({ url, branches, tags, ...startPathInfo }, { flowLevel: 1 }).trimRight()
       logger.info(`No matching references found for content source entry (${sourceInfo.replace(NEWLINE_RX, ' | ')})`)
-      return []
     }
-    return Promise.all(refs.map((it) => collectFilesFromReference(source, repo, remoteName, authStatus, it, originUrl)))
-  })
+  }
+  return startPaths
 }
 
 // QUESTION should we resolve HEAD to a ref eagerly to avoid having to do a match on it?
 async function selectReferences (source, repo, remote) {
   let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns } = source
-  const isBare = repo.noCheckout
+  const managed = 'url' in repo
+  const isBare = managed || repo.noCheckout
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
-  const noWorktree = repo.url ? undefined : false
+  const noWorktree = managed ? undefined : false
   const refs = new Map()
   if (
     tagPatterns &&
@@ -389,10 +393,11 @@ async function selectReferences (source, repo, remote) {
         refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
       }
     }
-  } else if (!remoteBranches.length) {
+  } else if (!managed || !remoteBranches.length) {
     const localBranches = await git.listBranches(repo)
     if (localBranches.length) {
       for (const shortname of filterRefs(localBranches, branchPatterns, patternCache)) {
+        if (refs.has(shortname)) continue // NOTE prefer remote branches in bare repository
         refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head: noWorktree })
       }
     }
@@ -413,10 +418,9 @@ function getCurrentBranchName (repo, remote) {
   ).then((ref) => (ref.startsWith('refs/') ? ref.replace(SHORTEN_REF_RX, '') : undefined))
 }
 
-async function collectFilesFromReference (source, repo, remoteName, authStatus, ref, originUrl) {
+async function selectStartPaths (source, repo, remoteName, ref) {
   const url = repo.url
   const displayUrl = url || repo.dir
-  const { version, editUrl } = source
   const worktreePath = ref.head
   if (!worktreePath) {
     ref.oid = await git.resolveRef(
@@ -436,17 +440,22 @@ async function collectFilesFromReference (source, repo, remoteName, authStatus,
       const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
       throw new Error(`no start paths found in ${where} (${ref.type}: ${ref.shortname}${flag})`)
     }
-    return Promise.all(
-      startPaths.map((startPath) =>
-        collectFilesFromStartPath(startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version)
-      )
-    )
+    return startPaths
   }
-  const startPath = cleanStartPath(coerceToString(source.startPath))
-  return collectFilesFromStartPath(startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version)
+  return [cleanStartPath(coerceToString(source.startPath))]
 }
 
-function collectFilesFromStartPath (startPath, repo, authStatus, ref, worktreePath, originUrl, editUrl, version) {
+async function collectFilesFromStartPaths (startPaths, repo, authStatus) {
+  const buckets = []
+  for (const { startPath, ref, originUrl, editUrl, version } of startPaths) {
+    buckets.push(await collectFilesFromStartPath(startPath, repo, authStatus, ref, originUrl, editUrl, version))
+  }
+  repo.cache = undefined
+  return buckets
+}
+
+function collectFilesFromStartPath (startPath, repo, authStatus, ref, originUrl, editUrl, version) {
+  const worktreePath = ref.head
   const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
   return (worktreePath ? readFilesFromWorktree(origin) : readFilesFromGitTree(repo, ref.oid, startPath))
     .then((files) =>
@@ -526,26 +535,25 @@ function srcFs (cwd, origin) {
 }
 
 function readFilesFromGitTree (repo, oid, startPath) {
-  return git
-    .readTree(Object.assign({ oid }, repo))
-    .then((root) =>
-      getGitTreeAtStartPath(repo, oid, startPath).then((start) =>
-        srcGitTree(repo, Object.assign(root, { dirname: '' }), start)
-      )
-    )
+  return git.readTree(Object.assign({ oid }, repo)).then((root) => {
+    Object.assign(root, { dirname: '' })
+    return startPath
+      ? getGitTreeAtStartPath(repo, oid, startPath).then((start) => {
+        Object.assign(start, { dirname: startPath })
+        return srcGitTree(repo, root, start)
+      })
+      : srcGitTree(repo, root)
+  })
 }
 
 function getGitTreeAtStartPath (repo, oid, startPath) {
-  return git.readTree(Object.assign({ oid, filepath: startPath }, repo)).then(
-    (result) => Object.assign(result, { dirname: startPath }),
-    (err) => {
-      const m = err instanceof ObjectTypeError && err.data.expected === 'tree' ? 'is not a directory' : 'does not exist'
-      throw new Error(`the start path '${startPath}' ${m}`)
-    }
-  )
+  return git.readTree(Object.assign({ oid, filepath: startPath }, repo)).catch((err) => {
+    const m = err instanceof ObjectTypeError && err.data.expected === 'tree' ? 'is not a directory' : 'does not exist'
+    throw new Error(`the start path '${startPath}' ${m}`)
+  })
 }
 
-function srcGitTree (repo, root, start) {
+function srcGitTree (repo, root, start = root) {
   return new Promise((resolve, reject) => {
     const files = []
     createGitTreeWalker(repo, root, filterGitEntry, gitEntryToFile)
@@ -832,7 +840,6 @@ function onGitProgress ({ phase, loaded, total }) {
 
 function onGitComplete (err) {
   if (err) {
-    // TODO could use progressBar.interrupt() to replace bar with message instead
     this.chars.incomplete = '?'
     this.update(0)
     // NOTE force progress bar to update regardless of throttle setting
@@ -982,8 +989,8 @@ function ensureCacheDir (preferredCacheDir, startDir) {
   )
 }
 
-function transformGitCloneError (err, displayUrl) {
-  let wrappedMsg, trimMessage
+function transformGitCloneError (err, displayUrl, authRequested) {
+  let wrappedMsg, recoverable, trimMessage
   if (HTTP_ERROR_CODE_RX.test(err.code)) {
     switch (err.data.statusCode) {
       case 401:
@@ -992,25 +999,26 @@ function transformGitCloneError (err, displayUrl) {
           : 'Content repository not found or requires credentials'
         break
       case 404:
-        wrappedMsg = 'Content repository not found'
+        wrappedMsg = authRequested
+          ? 'Content repository not found or credentials were rejected'
+          : 'Content repository not found'
         break
       default:
         wrappedMsg = err.message
-        trimMessage = true
+        recoverable = trimMessage = true
     }
   } else if (err instanceof UrlParseError || err instanceof UnknownTransportError) {
     wrappedMsg = 'Content source uses an unsupported transport protocol'
   } else if (err.code === 'ENOTFOUND') {
     wrappedMsg = `Content repository host could not be resolved: ${err.hostname}`
   } else {
-    wrappedMsg = `${err.name}: ${err.message}`
-    trimMessage = true
-  }
-  if (trimMessage) {
-    wrappedMsg = ~(wrappedMsg = wrappedMsg.trimRight()).indexOf('. ') ? wrappedMsg : wrappedMsg.replace(/\.$/, '')
+    wrappedMsg = err.message || String(err)
+    recoverable = trimMessage = true
   }
+  if (trimMessage && !~(wrappedMsg = wrappedMsg.trimEnd()).indexOf('. ')) wrappedMsg = wrappedMsg.replace(/\.$/, '')
   const errWrapper = new Error(`${wrappedMsg} (url: ${displayUrl})`)
-  errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
+  errWrapper.stack += `\nCaused by: ${err.stack ? inspect(err).replace(/^Error \[(.+?)\](?=: )/, '$1') : err}`
+  if (recoverable) Object.defineProperty(errWrapper, 'recoverable', { value: true })
   return errWrapper
 }
 
@@ -1041,11 +1049,11 @@ function coerceToString (value) {
 function resolveRepositoryFromWorktree (repo) {
   return fsp
     .readFile(repo.gitdir, 'utf8')
-    .then((contents) => contents.trimRight().substr(8))
+    .then((contents) => contents.substr(8).trimEnd())
     .then((worktreeGitdir) =>
       fsp.readFile(ospath.join(worktreeGitdir, 'commondir'), 'utf8').then(
         (contents) => {
-          const gitdir = ospath.join(worktreeGitdir, contents.trimRight())
+          const gitdir = ospath.join(worktreeGitdir, contents.trimEnd())
           const dir = ospath.basename(gitdir) === '.git' ? ospath.dirname(gitdir) : gitdir
           return Object.assign(repo, { dir, gitdir, worktreeName: ospath.basename(worktreeGitdir) })
         },
@@ -1077,7 +1085,7 @@ function findWorktrees (repo, patterns) {
                 .then((branch = worktreeName) =>
                   fsp
                     .readFile(ospath.join(gitdir, 'gitdir'), 'utf8')
-                    .then((contents) => [branch, { head: ospath.dirname(contents.trimRight()), name: worktreeName }])
+                    .then((contents) => [branch, { head: ospath.dirname(contents.trimEnd()), name: worktreeName }])
                 )
             })
           )
@@ -1086,4 +1094,37 @@ function findWorktrees (repo, patterns) {
   ).then((entries = []) => mainWorktree.then((entry) => new Map(entry ? entries.push(entry) && entries : entries)))
 }
 
+async function gracefulPromiseAllWithLimit (tasks, limit = Infinity) {
+  const rejections = []
+  const recordRejection = (err) => rejections.push(err) && undefined
+  const started = []
+  if (tasks.length <= limit) {
+    for (const task of tasks) started.push(task().catch(recordRejection))
+  } else {
+    const pending = []
+    for (const task of tasks) {
+      const current = task()
+        .catch(recordRejection)
+        .finally(() => pending.splice(pending.indexOf(current), 1))
+      started.push(current)
+      if (pending.push(current) < limit) continue
+      await Promise.race(pending)
+      if (rejections.length) break
+    }
+  }
+  return Promise.all(started).then((results) => [results, rejections])
+}
+
+async function promiseAllWithLimit (tasks, limit = Infinity) {
+  if (tasks.length <= limit) return Promise.all(tasks.map((task) => task()))
+  const started = []
+  const pending = []
+  for (const task of tasks) {
+    const current = task().finally(() => pending.splice(pending.indexOf(current), 1))
+    started.push(current)
+    if (pending.push(current) >= limit) await Promise.race(pending)
+  }
+  return Promise.all(started)
+}
+
 module.exports = aggregateContent

package/lib/git-plugin-http.js

@@ -49,11 +49,12 @@ module.exports = ({ headers: extraHeaders, httpProxy, httpsProxy, noProxy } = {}
   }
   return {
     async request ({ url, method, headers, body }) {
-      headers = mergeHeaders(headers, extraHeaders)
+      headers = Object.assign(mergeHeaders(headers, extraHeaders), { connection: 'close' })
       body = await mergeBuffers(body)
-      return new Promise((resolve, reject) =>
-        get({ url, method, headers, body }, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
-      )
+      return new Promise((resolve, reject) => {
+        const opts = { url, method, headers, body, timeout: 0, keepAlive: false }
+        return get(opts, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
+      })
     },
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.2.0-alpha.2",
+  "version": "3.2.0-alpha.4",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -29,13 +29,13 @@
   },
   "dependencies": {
     "@antora/expand-path-helper": "~2.0",
-    "@antora/logger": "3.2.0-alpha.2",
+    "@antora/logger": "3.2.0-alpha.4",
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
     "glob-stream": "~7.0",
     "hpagent": "~1.2",
-    "isomorphic-git": "~1.21",
+    "isomorphic-git": "~1.25",
     "js-yaml": "~4.1",
     "multi-progress": "~4.0",
     "picomatch": "~2.3",
@@ -62,7 +62,7 @@
   ],
   "scripts": {
     "test": "_mocha",
-    "prepublishOnly": "node $npm_config_local_prefix/npm/prepublishOnly.js",
-    "postpublish": "node $npm_config_local_prefix/npm/postpublish.js"
+    "prepublishOnly": "npx -y downdoc@latest --prepublish",
+    "postpublish": "npx -y downdoc@latest --postpublish"
   }
 }