@antora/content-aggregator 3.1.6 → 3.1.8

package/lib/aggregate-content.js

@@ -15,7 +15,8 @@ const getCacheDir = require('cache-directory')
 const GitCredentialManagerStore = require('./git-credential-manager-store')
 const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
-const globStream = require('glob-stream')
+const { globStream } = require('fast-glob')
+const { inspect } = require('util')
 const invariably = require('./invariably')
 const logger = require('./logger')
 const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')
@@ -118,11 +119,11 @@ async function collectFiles (sourcesByUrl, loadOpts, concurrency, fetchedUrls) {
   })
   return gracefulPromiseAllWithLimit(loadTasks, concurrency.fetch).then(([results, rejections]) => {
     if (rejections.length) {
-      if (concurrency.fetch > 1 && rejections.every(({ recoverable }) => recoverable)) {
+      if (concurrency.fetch > 1 && results.length > 1 && rejections.every(({ recoverable }) => recoverable)) {
         if (loadOpts.progress) loadOpts.progress.terminate() // reset cursor position and allow it be reused
-        const msg0 = 'An unexpected error occurred while concurrently fetching content sources.'
+        const msg0 = 'An unexpected error occurred while fetching content sources concurrently.'
         const msg1 = 'Retrying with git.fetch_concurrency value of 1.'
-        logger.warn(msg0 + ' ' + msg1)
+        logger.warn(rejections[0], msg0 + ' ' + msg1)
         const fulfilledUrls = results.map((it) => it && it.repo.url && it.url).filter((it) => it)
         return collectFiles(sourcesByUrl, loadOpts, Object.assign(concurrency, { fetch: 1 }), fulfilledUrls)
       }
@@ -383,7 +384,7 @@ async function selectReferences (source, repo, remote) {
 }
 
 /**
- * Returns the current branch name unless the HEAD is detached.
+ * Returns the current branch name or undefined if the HEAD is detached.
  */
 function getCurrentBranchName (repo, remote) {
   let refPromise
@@ -437,12 +438,13 @@ function collectFilesFromStartPath (startPath, repo, authStatus, ref, originUrl,
   const worktreePath = ref.head
   const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
   return (worktreePath ? readFilesFromWorktree(origin) : readFilesFromGitTree(repo, ref.oid, startPath))
-    .then((files) =>
-      Object.assign(deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version))), {
-        files: files.map((file) => assignFileProperties(file, origin)),
-        origins: [origin],
-      })
-    )
+    .then((files) => {
+      const batch = deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version)))
+      if ('nav' in batch) batch.nav.origin = origin
+      batch.files = files.map((file) => assignFileProperties(file, origin))
+      batch.origins = [origin]
+      return batch
+    })
     .catch((err) => {
       const where = worktreePath || (worktreePath === false ? repo.gitdir : repo.url || repo.dir)
       const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
@@ -467,19 +469,18 @@ function readFilesFromWorktree (origin) {
 }
 
 function srcFs (cwd, origin) {
-  return new Promise((resolve, reject, cache = Object.create(null), files = [], relpathStart = cwd.length + 1) =>
+  return new Promise((resolve, reject, files = []) =>
     pipeline(
-      globStream(CONTENT_SRC_GLOB, Object.assign({ cache, cwd }, CONTENT_SRC_OPTS)),
-      forEach(({ path: abspathPosix }, _, done) => {
-        if ((cache[abspathPosix] || {}).constructor === Array) return done() // detects some directories
-        const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
-        const relpath = abspath.substr(relpathStart)
-        symlinkAwareStat(abspath).then(
-          (stat) => {
-            if (stat.isDirectory()) return done() // detects directories that slipped through cache check
+      globStream(CONTENT_SRC_GLOB, Object.assign({ cwd }, CONTENT_SRC_OPTS)),
+      forEach(({ path: relpath, dirent }, _, done) => {
+        if (dirent.isDirectory()) return done()
+        const relpathPosix = relpath
+        const abspath = posixify ? ospath.join(cwd, (relpath = ospath.normalize(relpath))) : cwd + '/' + relpath
+        fsp.stat(abspath).then(
+          (stat) =>
             fsp.readFile(abspath).then(
               (contents) => {
-                files.push(new File({ path: posixify ? posixify(relpath) : relpath, contents, stat, src: { abspath } }))
+                files.push(new File({ path: relpathPosix, contents, stat, src: { abspath } }))
                 done()
               },
               (readErr) => {
@@ -489,22 +490,28 @@ function srcFs (cwd, origin) {
                   : logger.error(logObject, readErr.message.replace(`'${abspath}'`, relpath))
                 done()
               }
-            )
-          },
+            ),
           (statErr) => {
             const logObject = { file: { abspath, origin } }
-            if (statErr.symlink) {
-              logger.error(
-                logObject,
-                (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
-                  `${relpath} -> ${statErr.symlink}`
-              )
-            } else if (statErr.code === 'ENOENT') {
-              logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
+            if (dirent.isSymbolicLink()) {
+              fsp
+                .readlink(abspath)
+                .then(
+                  (symlink) =>
+                    (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
+                    `${relpath} -> ${symlink}`,
+                  () => statErr.message.replace(`'${abspath}'`, relpath)
+                )
+                .then((message) => {
+                  logger.error(logObject, message)
+                  done()
+                })
             } else {
-              logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
+              statErr.code === 'ENOENT'
+                ? logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
+                : logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
+              done()
             }
-            done()
           }
         )
       }),
@@ -674,8 +681,13 @@ function filterGitEntry (entry) {
 
 function gitEntryToFile (entry) {
   return git.readBlob(entry).then(({ blob: contents }) => {
-    contents = Buffer.from(contents.buffer)
-    const stat = Object.assign(new fs.Stats(), { mode: entry.mode, mtime: undefined, size: contents.byteLength })
+    const stat = {
+      mode: entry.mode,
+      size: (contents = Buffer.from(contents.buffer)).byteLength,
+      isDirectory: invariably.false,
+      isFile: invariably.true,
+      isSymbolicLink: invariably.false,
+    }
     return new File({ path: entry.path, contents, stat })
   })
 }
@@ -687,7 +699,7 @@ function loadComponentDescriptor (files, ref, version) {
   files.splice(descriptorFileIdx, 1)
   let data
   try {
-    data = yaml.load(descriptorFile.contents.toString(), { schema: yaml.CORE_SCHEMA })
+    data = Object(yaml.load(descriptorFile.contents.toString(), { schema: yaml.CORE_SCHEMA }))
   } catch (err) {
     throw Object.assign(err, { message: `${COMPONENT_DESC_FILENAME} has invalid syntax; ${err.message}` })
   }
@@ -902,20 +914,6 @@ function isDirectory (url) {
   return fsp.stat(url).then((stat) => stat.isDirectory(), invariably.false)
 }
 
-function symlinkAwareStat (path_) {
-  return fsp.lstat(path_).then((lstat) => {
-    if (!lstat.isSymbolicLink()) return lstat
-    return fsp.stat(path_).catch((statErr) =>
-      fsp
-        .readlink(path_)
-        .catch(invariably.void)
-        .then((symlink) => {
-          throw Object.assign(statErr, { symlink })
-        })
-    )
-  })
-}
-
 function tagsSpecified (sources) {
   return sources.some(({ tags }) => tags && (Array.isArray(tags) ? tags.length : true))
 }
@@ -986,15 +984,14 @@ function transformGitCloneError (err, displayUrl, authRequested) {
   } else if (err.code === 'ENOTFOUND') {
     wrappedMsg = `Content repository host could not be resolved: ${err.hostname}`
   } else {
-    wrappedMsg = `${err.name}: ${err.message}`
+    wrappedMsg = err.message || String(err)
     recoverable = trimMessage = true
   }
-  if (trimMessage) {
-    wrappedMsg = ~(wrappedMsg = wrappedMsg.trimEnd()).indexOf('. ') ? wrappedMsg : wrappedMsg.replace(/\.$/, '')
-  }
+  if (trimMessage && !~(wrappedMsg = wrappedMsg.trimEnd()).indexOf('. ')) wrappedMsg = wrappedMsg.replace(/\.$/, '')
   const errWrapper = new Error(`${wrappedMsg} (url: ${displayUrl})`)
-  errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
-  return recoverable ? Object.assign(errWrapper, { recoverable }) : errWrapper
+  errWrapper.stack += `\nCaused by: ${err.stack ? inspect(err).replace(/^Error \[(.+?)\](?=: )/, '$1') : err}`
+  if (recoverable) Object.defineProperty(errWrapper, 'recoverable', { value: true })
+  return errWrapper
 }
 
 function splitRefPatterns (str) {

package/lib/constants.js

@@ -3,8 +3,8 @@
 module.exports = Object.freeze({
   COMPONENT_DESC_FILENAME: 'antora.yml',
   CONTENT_CACHE_FOLDER: 'content',
-  CONTENT_SRC_GLOB: '**/*[!~]',
-  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, strict: false },
+  CONTENT_SRC_GLOB: '**/!(*~)',
+  CONTENT_SRC_OPTS: { dot: true, ignore: ['**/.*{,/**}'], objectMode: true, onlyFiles: false, unique: false },
   FILE_MODES: { 100644: 0o100666 & ~process.umask(), 100755: 0o100777 & ~process.umask() },
   GIT_CORE: 'antora',
   GIT_OPERATION_LABEL_LENGTH: 8,

package/lib/git-plugin-http.js

@@ -49,11 +49,12 @@ module.exports = ({ headers: extraHeaders, httpProxy, httpsProxy, noProxy } = {}
   }
   return {
     async request ({ url, method, headers, body }) {
-      headers = mergeHeaders(headers, extraHeaders)
+      headers = Object.assign(mergeHeaders(headers, extraHeaders), { connection: 'close' })
       body = await mergeBuffers(body)
-      return new Promise((resolve, reject) =>
-        get({ url, method, headers, body }, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
-      )
+      return new Promise((resolve, reject) => {
+        const opts = { url, method, headers, body, timeout: 0, keepAlive: false }
+        return get(opts, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
+      })
     },
   }
 }

package/lib/matcher.js

@@ -16,15 +16,10 @@ const BASE_OPTS = {
   strictSlashes: true,
 }
 
-function makeMatcherRx (input, opts) {
-  if (input && ~input.indexOf('{')) input = input.replace(/^([^({]+)\./, '$1(?:.)')
-  return makeRe(input, opts)
-}
-
 module.exports = {
   MATCH_ALL_RX: { test: () => true },
   expandBraces,
-  makeMatcherRx,
+  makeMatcherRx: makeRe,
   pathMatcherOpts: Object.assign({}, BASE_OPTS, { dot: false }),
   refMatcherOpts: (cache) =>
     Object.assign({}, BASE_OPTS, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.1.6",
+  "version": "3.1.8",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -29,20 +29,20 @@
   },
   "dependencies": {
     "@antora/expand-path-helper": "~2.0",
-    "@antora/logger": "3.1.6",
+    "@antora/logger": "3.1.8",
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "glob-stream": "~7.0",
+    "fast-glob": "~3.3",
     "hpagent": "~1.2",
-    "isomorphic-git": "~1.21",
+    "isomorphic-git": "~1.25",
     "js-yaml": "~4.1",
     "multi-progress": "~4.0",
-    "picomatch": "~2.3",
+    "picomatch": "~4.0",
     "progress": "~2.0",
     "should-proxy": "~1.0",
     "simple-get": "~4.0",
-    "vinyl": "~2.2"
+    "vinyl": "~3.0"
   },
   "engines": {
     "node": ">=16.0.0"