@antora/content-aggregator 3.2.0-alpha.4 → 3.2.0-alpha.6

package/lib/aggregate-content.js

@@ -1,32 +1,32 @@
 'use strict'
 
 const computeOrigin = require('./compute-origin')
-const { createHash } = require('crypto')
+const { createHash } = require('node:crypto')
 const createGitHttpPlugin = require('./git-plugin-http')
 const decodeUint8Array = require('./decode-uint8-array')
 const deepClone = require('./deep-clone')
-const EventEmitter = require('events')
+const EventEmitter = require('node:events')
 const expandPath = require('@antora/expand-path-helper')
 const File = require('./file')
 const filterRefs = require('./filter-refs')
-const fs = require('fs')
+const fs = require('node:fs')
 const { promises: fsp } = fs
 const getCacheDir = require('cache-directory')
 const GitCredentialManagerStore = require('./git-credential-manager-store')
 const git = require('./git')
 const { NotFoundError, ObjectTypeError, UnknownTransportError, UrlParseError } = git.Errors
-const globStream = require('glob-stream')
-const { inspect } = require('util')
+const { globStream } = require('fast-glob')
+const { inspect } = require('node:util')
 const invariably = require('./invariably')
 const logger = require('./logger')
 const { makeMatcherRx, versionMatcherOpts: VERSION_MATCHER_OPTS } = require('./matcher')
 const MultiProgress = require('multi-progress') // calls require('progress') as a peer dependencies
-const ospath = require('path')
+const ospath = require('node:path')
 const { posix: path } = ospath
 const posixify = require('./posixify')
 const removeGitSuffix = require('./remove-git-suffix')
 const { fs: resolvePathGlobsFs, git: resolvePathGlobsGit } = require('./resolve-path-globs')
-const { pipeline, Writable } = require('stream')
+const { pipeline, Writable } = require('node:stream')
 const forEach = (write) => new Writable({ objectMode: true, write })
 const userRequire = require('@antora/user-require-helper')
 const yaml = require('js-yaml')
@@ -102,7 +102,8 @@ function aggregateContent (playbook) {
     }, new Map())
     const progress = !quiet && createProgress(sourcesByUrl.keys(), process.stdout)
     const refPatternCache = Object.assign(new Map(), { braces: new Map() })
-    const loadOpts = { cacheDir, fetch, gitPlugins, progress, startDir, refPatternCache }
+    const fetchConfig = { always: fetch, depth: Math.max(0, gitConfig.fetchDepth ?? 1) }
+    const loadOpts = { cacheDir, fetch: fetchConfig, gitPlugins, progress, startDir, refPatternCache }
     return collectFiles(sourcesByUrl, loadOpts, concurrency).then(buildAggregate, (err) => {
       progress && progress.terminate()
       throw err
@@ -110,11 +111,11 @@ function aggregateContent (playbook) {
   })
 }
 
-async function collectFiles (sourcesByUrl, loadOpts, concurrency, fetchedUrls) {
+async function collectFiles (sourcesByUrl, loadOpts, concurrency, fetchedUrls = []) {
   const loadTasks = [...sourcesByUrl.entries()].map(([url, sources]) => {
     const loadOptsForUrl = Object.assign({}, loadOpts)
-    if (loadOpts.fetch && fetchedUrls && fetchedUrls.length && fetchedUrls.includes(url)) loadOptsForUrl.fetch = false
-    if (tagsSpecified(sources)) loadOptsForUrl.fetchTags = true
+    if (loadOpts.fetch.always && fetchedUrls.length && fetchedUrls.includes(url)) loadOptsForUrl.fetch.always = false
+    if (tagsSpecified(sources)) loadOptsForUrl.fetch.tags = true
     return loadRepository.bind(null, url, loadOptsForUrl, { url, sources })
   })
   return gracefulPromiseAllWithLimit(loadTasks, concurrency.fetch).then(([results, rejections]) => {
@@ -163,7 +164,7 @@ async function loadRepository (url, opts, result = {}) {
   if (~url.indexOf(':') && GIT_URI_DETECTOR_RX.test(url)) {
     let credentials, displayUrl
     ;({ displayUrl, url, credentials } = extractCredentials(url))
-    const { cacheDir, fetch, fetchTags, gitPlugins, progress } = opts
+    const { cacheDir, fetch, gitPlugins, progress } = opts
     dir = ospath.join(cacheDir, generateCloneFolderName(displayUrl))
     // NOTE the presence of the url property on the repo object implies the repository is remote
     repo = { cache, dir, fs, gitdir: dir, noCheckout: true, url }
@@ -171,9 +172,9 @@ async function loadRepository (url, opts, result = {}) {
     const validStateFile = ospath.join(dir, VALID_STATE_FILENAME)
     try {
       await fsp.access(validStateFile)
-      if (fetch) {
+      if (fetch.always) {
         await fsp.unlink(validStateFile)
-        const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetchTags, 'fetch')
+        const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetch, 'fetch')
         await git
           .fetch(fetchOpts)
           .then(() => {
@@ -193,7 +194,7 @@ async function loadRepository (url, opts, result = {}) {
     } catch (gitErr) {
       await fsp['rm' in fsp ? 'rm' : 'rmdir'](dir, { recursive: true, force: true })
       if (gitErr.rethrow) throw transformGitCloneError(gitErr, displayUrl)
-      const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetchTags, 'clone')
+      const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetch, 'clone')
       await git
         .clone(fetchOpts)
         .then(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo)))
@@ -325,9 +326,9 @@ async function selectReferences (source, repo, remote) {
   } else {
     worktreePatterns = worktreePatterns === undefined ? [worktreeName || '.'] : []
   }
+  let currentBranch
   if (branchPatterns.length === 1 && (branchPatterns[0] === 'HEAD' || branchPatterns[0] === '.')) {
-    const currentBranch = await getCurrentBranchName(repo, remote)
-    if (currentBranch) {
+    if ((currentBranch = await getCurrentBranchName(repo, remote).then((branch) => branch ?? false))) {
       branchPatterns = [currentBranch]
     } else if (isBare) {
       return [...refs.values()]
@@ -341,9 +342,7 @@ async function selectReferences (source, repo, remote) {
     let headBranchIdx
     // NOTE we can assume at least two entries if HEAD or . are present
     if (~(headBranchIdx = branchPatterns.indexOf('HEAD')) || ~(headBranchIdx = branchPatterns.indexOf('.'))) {
-      const currentBranch = await getCurrentBranchName(repo, remote)
-      if (currentBranch) {
-        // NOTE ignore if current branch is already in list
+      if ((currentBranch = await getCurrentBranchName(repo, remote).then((branch) => branch ?? false))) {
         if (~branchPatterns.indexOf(currentBranch)) {
           branchPatterns.splice(headBranchIdx, 1)
         } else {
@@ -373,7 +372,11 @@ async function selectReferences (source, repo, remote) {
   }
   // NOTE only consider local branches if repo has a worktree or there are no remote tracking branches
   if (!isBare) {
-    const localBranches = await git.listBranches(repo)
+    const localBranches = await git.listBranches(repo).then((branches) => {
+      if (branches.length) return branches
+      if (currentBranch == null) return getCurrentBranchName(repo).then((branch) => (branch ? [branch] : []))
+      return currentBranch ? [currentBranch] : []
+    })
     if (localBranches.length) {
       const worktrees = await findWorktrees(repo, worktreePatterns)
       let onMatch
@@ -406,11 +409,11 @@ async function selectReferences (source, repo, remote) {
 }
 
 /**
- * Returns the current branch name unless the HEAD is detached.
+ * Returns the current branch name or undefined if the HEAD is detached.
  */
 function getCurrentBranchName (repo, remote) {
   return (
-    repo.noCheckout && remote
+    remote && repo.noCheckout
       ? git
         .resolveRef(Object.assign({ ref: 'refs/remotes/' + remote + '/HEAD', depth: 2 }, repo))
         .catch(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 2 }, repo)))
@@ -458,12 +461,13 @@ function collectFilesFromStartPath (startPath, repo, authStatus, ref, originUrl,
   const worktreePath = ref.head
   const origin = computeOrigin(originUrl, authStatus, repo.gitdir, ref, startPath, worktreePath, editUrl)
   return (worktreePath ? readFilesFromWorktree(origin) : readFilesFromGitTree(repo, ref.oid, startPath))
-    .then((files) =>
-      Object.assign(deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version))), {
-        files: files.map((file) => assignFileProperties(file, origin)),
-        origins: [origin],
-      })
-    )
+    .then((files) => {
+      const batch = deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version)))
+      if ('nav' in batch) batch.nav.origin = origin
+      batch.files = files.map((file) => assignFileProperties(file, origin))
+      batch.origins = [origin]
+      return batch
+    })
     .catch((err) => {
       const where = worktreePath || (worktreePath === false ? repo.gitdir : repo.url || repo.dir)
       const flag = worktreePath ? ' <worktree>' : ref.remote && worktreePath === false ? ` <remotes/${ref.remote}>` : ''
@@ -488,19 +492,18 @@ function readFilesFromWorktree (origin) {
 }
 
 function srcFs (cwd, origin) {
-  return new Promise((resolve, reject, cache = Object.create(null), files = [], relpathStart = cwd.length + 1) =>
+  return new Promise((resolve, reject, files = []) =>
     pipeline(
-      globStream(CONTENT_SRC_GLOB, Object.assign({ cache, cwd }, CONTENT_SRC_OPTS)),
-      forEach(({ path: abspathPosix }, _, done) => {
-        if ((cache[abspathPosix] || {}).constructor === Array) return done() // detects some directories
-        const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
-        const relpath = abspath.substr(relpathStart)
-        symlinkAwareStat(abspath).then(
-          (stat) => {
-            if (stat.isDirectory()) return done() // detects directories that slipped through cache check
+      globStream(CONTENT_SRC_GLOB, Object.assign({ cwd }, CONTENT_SRC_OPTS)),
+      forEach(({ path: relpath, dirent }, _, done) => {
+        if (dirent.isDirectory()) return done()
+        const relpathPosix = relpath
+        const abspath = posixify ? ospath.join(cwd, (relpath = ospath.normalize(relpath))) : cwd + '/' + relpath
+        fsp.stat(abspath).then(
+          (stat) =>
             fsp.readFile(abspath).then(
               (contents) => {
-                files.push(new File({ path: posixify ? posixify(relpath) : relpath, contents, stat, src: { abspath } }))
+                files.push(new File({ path: relpathPosix, contents, stat, src: { abspath } }))
                 done()
               },
               (readErr) => {
@@ -510,22 +513,28 @@ function srcFs (cwd, origin) {
                   : logger.error(logObject, readErr.message.replace(`'${abspath}'`, relpath))
                 done()
               }
-            )
-          },
+            ),
           (statErr) => {
             const logObject = { file: { abspath, origin } }
-            if (statErr.symlink) {
-              logger.error(
-                logObject,
-                (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
-                  `${relpath} -> ${statErr.symlink}`
-              )
-            } else if (statErr.code === 'ENOENT') {
-              logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
+            if (dirent.isSymbolicLink()) {
+              fsp
+                .readlink(abspath)
+                .then(
+                  (symlink) =>
+                    (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
+                    `${relpath} -> ${symlink}`,
+                  () => statErr.message.replace(`'${abspath}'`, relpath)
+                )
+                .then((message) => {
+                  logger.error(logObject, message)
+                  done()
+                })
             } else {
-              logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
+              statErr.code === 'ENOENT'
+                ? logger.warn(logObject, `ENOENT: file or directory disappeared, ${statErr.syscall} ${relpath}`)
+                : logger.error(logObject, statErr.message.replace(`'${abspath}'`, relpath))
+              done()
             }
-            done()
           }
         )
       }),
@@ -695,8 +704,13 @@ function filterGitEntry (entry) {
 
 function gitEntryToFile (entry) {
   return git.readBlob(entry).then(({ blob: contents }) => {
-    contents = Buffer.from(contents.buffer)
-    const stat = Object.assign(new fs.Stats(), { mode: entry.mode, mtime: undefined, size: contents.byteLength })
+    const stat = {
+      mode: entry.mode,
+      size: (contents = Buffer.from(contents.buffer)).byteLength,
+      isDirectory: invariably.false,
+      isFile: invariably.true,
+      isSymbolicLink: invariably.false,
+    }
     return new File({ path: entry.path, contents, stat })
   })
 }
@@ -708,7 +722,7 @@ function loadComponentDescriptor (files, ref, version) {
   files.splice(descriptorFileIdx, 1)
   let data
   try {
-    data = yaml.load(descriptorFile.contents.toString(), { schema: yaml.CORE_SCHEMA })
+    data = Object(yaml.load(descriptorFile.contents.toString(), { schema: yaml.CORE_SCHEMA }))
   } catch (err) {
     throw Object.assign(err, { message: `${COMPONENT_DESC_FILENAME} has invalid syntax; ${err.message}` })
   }
@@ -764,18 +778,20 @@ function assignFileProperties (file, origin) {
   return file
 }
 
-function buildFetchOptions (repo, progress, displayUrl, credentialsFromUrl, gitPlugins, fetchTags, operation) {
+function buildFetchOptions (repo, progress, displayUrl, credentialsFromUrl, gitPlugins, fetch, operation) {
   const { credentialManager, http, urlRouter } = gitPlugins
+  const corsProxy = false
+  const depth = fetch.depth || undefined
   const onAuth = resolveCredentials.bind(credentialManager, new Map().set(undefined, credentialsFromUrl))
   const onAuthFailure = onAuth
   const onAuthSuccess = (url) => credentialManager.approved({ url })
-  const opts = Object.assign({ corsProxy: false, depth: 1, http, onAuth, onAuthFailure, onAuthSuccess }, repo)
+  const opts = Object.assign({ corsProxy, depth, http, onAuth, onAuthFailure, onAuthSuccess }, repo)
   if (urlRouter) opts.url = urlRouter.ensureGitSuffix(opts.url)
   if (progress) opts.onProgress = createProgressListener(progress, displayUrl, operation)
   if (operation === 'fetch') {
     opts.prune = true
-    if (fetchTags) opts.tags = opts.pruneTags = true
-  } else if (!fetchTags) {
+    if (fetch.tags) opts.tags = opts.pruneTags = true
+  } else if (!fetch.tags) {
     opts.noTags = true
   }
   return opts
@@ -884,7 +900,7 @@ function identifyAuthStatus (credentialManager, credentials, url) {
  * The purpose of this function is generate a safe, unique folder name for the cloned
  * repository that gets stored in the cache directory.
  *
- * The generated folder name follows the pattern: <basename>-<sha1>-<version>.git
+ * The generated folder name follows the pattern: <basename>-<sha1-of-normalized-url>.git
  *
  * @param {String} url - The repository URL to convert.
  * @returns {String} The generated folder name.
@@ -928,20 +944,6 @@ function isDirectory (url) {
   return fsp.stat(url).then((stat) => stat.isDirectory(), invariably.false)
 }
 
-function symlinkAwareStat (path_) {
-  return fsp.lstat(path_).then((lstat) => {
-    if (!lstat.isSymbolicLink()) return lstat
-    return fsp.stat(path_).catch((statErr) =>
-      fsp
-        .readlink(path_)
-        .catch(invariably.void)
-        .then((symlink) => {
-          throw Object.assign(statErr, { symlink })
-        })
-    )
-  })
-}
-
 function tagsSpecified (sources) {
   return sources.some(({ tags }) => tags && (Array.isArray(tags) ? tags.length : true))
 }
@@ -1066,7 +1068,7 @@ function findWorktrees (repo, patterns) {
   if (!patterns.length) return new Map()
   const mainWorktree =
     patterns[0] === '.' && (patterns = patterns.slice(1))
-      ? getCurrentBranchName(repo).then((branch) => (branch ? [branch, { head: repo.dir, name: '.' }] : undefined))
+      ? getCurrentBranchName(repo).then((branch) => branch && [branch, { head: repo.dir, name: '.' }])
       : Promise.resolve()
   const worktreesDir = patterns.length ? ospath.join(repo.dir, '.git', 'worktrees') : undefined
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
@@ -1080,13 +1082,11 @@ function findWorktrees (repo, patterns) {
             worktreeNames.map((worktreeName) => {
               const gitdir = ospath.resolve(worktreesDir, worktreeName)
               // NOTE branch name defaults to worktree name if HEAD is detached
-              return git
-                .currentBranch(Object.assign({}, repo, { gitdir }))
-                .then((branch = worktreeName) =>
-                  fsp
-                    .readFile(ospath.join(gitdir, 'gitdir'), 'utf8')
-                    .then((contents) => [branch, { head: ospath.dirname(contents.trimEnd()), name: worktreeName }])
-                )
+              return getCurrentBranchName(Object.assign({}, repo, { gitdir })).then((branch = worktreeName) =>
+                fsp
+                  .readFile(ospath.join(gitdir, 'gitdir'), 'utf8')
+                  .then((contents) => [branch, { head: ospath.dirname(contents.trimEnd()), name: worktreeName }])
+              )
             })
           )
         )

package/lib/compute-origin.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const { posix: path } = require('path')
+const { posix: path } = require('node:path')
 const posixify = require('./posixify')
 const removeGitSuffix = require('./remove-git-suffix')
 

package/lib/constants.js

@@ -3,8 +3,8 @@
 module.exports = Object.freeze({
   COMPONENT_DESC_FILENAME: 'antora.yml',
   CONTENT_CACHE_FOLDER: 'content',
-  CONTENT_SRC_GLOB: '**/*[!~]',
-  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, strict: false },
+  CONTENT_SRC_GLOB: '**/!(*~)',
+  CONTENT_SRC_OPTS: { dot: true, ignore: ['**/.*{,/**}'], objectMode: true, onlyFiles: false, unique: false },
   FILE_MODES: { 100644: 0o100666 & ~process.umask(), 100755: 0o100777 & ~process.umask() },
   GIT_CORE: 'antora',
   GIT_OPERATION_LABEL_LENGTH: 8,

package/lib/decode-uint8-array.js

@@ -3,4 +3,4 @@
 module.exports = (({ StringDecoder }) => {
   const decoder = new StringDecoder()
   return decoder.write.bind(decoder)
-})(require('string_decoder'))
+})(require('node:string_decoder'))

package/lib/git-credential-manager-store.js

@@ -1,10 +1,10 @@
 'use strict'
 
-const { homedir } = require('os')
+const { homedir } = require('node:os')
 const expandPath = require('@antora/expand-path-helper')
-const { promises: fsp } = require('fs')
+const { promises: fsp } = require('node:fs')
 const invariably = require('./invariably')
-const ospath = require('path')
+const ospath = require('node:path')
 
 class GitCredentialManagerStore {
   configure ({ config, startDir }) {

package/lib/git.js

@@ -1,3 +1,10 @@
 'use strict'
 
-module.exports = require('isomorphic-git')
+const zlib = require('node:zlib')
+const { promisify } = require('node:util')
+
+module.exports = ((pakoModuleId) => {
+  const git = require('isomorphic-git')
+  require(pakoModuleId).inflate = promisify(zlib.inflate)
+  return git
+})('pako')

package/lib/matcher.js

@@ -16,15 +16,10 @@ const BASE_OPTS = {
   strictSlashes: true,
 }
 
-function makeMatcherRx (input, opts) {
-  if (input && ~input.indexOf('{')) input = input.replace(/^([^({]+)\./, '$1(?:.)')
-  return makeRe(input, opts)
-}
-
 module.exports = {
   MATCH_ALL_RX: Object.defineProperty({ test: () => true }, 'pattern', { value: '*' }),
   expandBraces,
-  makeMatcherRx,
+  makeMatcherRx: makeRe,
   pathMatcherOpts: Object.assign({}, BASE_OPTS, { dot: false }),
   refMatcherOpts: (cache) =>
     Object.assign({}, BASE_OPTS, {

package/lib/posixify.js

@@ -1,3 +1,3 @@
 'use strict'
 
-module.exports = require('path').sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
+module.exports = require('node:path').sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined

package/lib/resolve-path-globs.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const deepFlatten = require('./deep-flatten')
-const { promises: fsp } = require('fs')
+const { promises: fsp } = require('node:fs')
 const git = require('./git')
 const invariably = require('./invariably')
 const { expandBraces, makeMatcherRx, pathMatcherOpts: MATCHER_OPTS } = require('./matcher')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.2.0-alpha.4",
+  "version": "3.2.0-alpha.6",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -29,23 +29,23 @@
   },
   "dependencies": {
     "@antora/expand-path-helper": "~2.0",
-    "@antora/logger": "3.2.0-alpha.4",
+    "@antora/logger": "3.2.0-alpha.6",
     "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "glob-stream": "~7.0",
+    "fast-glob": "~3.3",
     "hpagent": "~1.2",
     "isomorphic-git": "~1.25",
     "js-yaml": "~4.1",
     "multi-progress": "~4.0",
-    "picomatch": "~2.3",
+    "picomatch": "~4.0",
     "progress": "~2.0",
     "should-proxy": "~1.0",
     "simple-get": "~4.0",
-    "vinyl": "~2.2"
+    "vinyl": "~3.0"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.0.0"
   },
   "files": [
     "lib/"