@antora/content-aggregator 3.2.0-alpha.10 → 3.2.0-alpha.12

package/lib/aggregate-content.js

@@ -116,6 +116,8 @@ async function collectFiles (sourcesByUrl, loadOpts, concurrency, fetchedUrls =
     const loadOptsForUrl = Object.assign({}, loadOpts)
     if (loadOpts.fetch.always && fetchedUrls.length && ~fetchedUrls.indexOf(url)) loadOptsForUrl.fetch.always = false
     if (tagsSpecified(sources)) loadOptsForUrl.fetch.tags = true
+    const commits = commitsRequested(sources)
+    if (commits) loadOptsForUrl.fetch.commits = commits
     return loadRepository.bind(null, url, loadOptsForUrl, { url, sources })
   })
   return gracefulPromiseAllWithLimit(loadTasks, concurrency.fetch).then(([results, rejections]) => {
@@ -177,6 +179,7 @@ async function loadRepository (url, opts, result = {}) {
         const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetch, 'fetch')
         await git
           .fetch(fetchOpts)
+          .then(() => ensureOids(fetchOpts))
           .then(() => {
             authStatus = identifyAuthStatus(credentialManager, credentials, url)
             return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
@@ -192,12 +195,13 @@ async function loadRepository (url, opts, result = {}) {
         authStatus = await git.getConfig(Object.assign({ path: 'remote.origin.private' }, repo))
       }
     } catch (gitErr) {
-      await fsp['rm' in fsp ? 'rm' : 'rmdir'](dir, { recursive: true, force: true })
+      await fsp.rm(dir, { recursive: true, force: true })
       if (gitErr.rethrow) throw transformGitCloneError(gitErr, displayUrl)
       const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetch, 'clone')
       await git
         .clone(fetchOpts)
         .then(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo)))
+        .then(() => ensureOids(fetchOpts))
         .then(() => {
           authStatus = identifyAuthStatus(credentialManager, credentials, url)
           return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
@@ -216,6 +220,7 @@ async function loadRepository (url, opts, result = {}) {
     if (dotgitStat.isDirectory()) {
       repo = { cache, dir, fs, gitdir: dotgit }
     } else if (dotgitStat.isFile()) {
+      // NOTE isomorphic-git will discover the gitdir, but we must do it eagerly to process worktree patterns correctly
       repo = await resolveRepositoryFromWorktree({ cache, dir, fs, gitdir: dotgit })
     } else {
       repo = { cache, dir, fs, gitdir: dir, noCheckout: true }
@@ -279,12 +284,12 @@ async function selectStartPathsForRepository (repo, sources) {
         }
       }
     } else {
-      const { url, branches, tags } = source
+      const { url, branches, tags, commits } = source
       const startPathInfo =
         'startPaths' in source
           ? { 'start paths': source.startPaths || undefined }
           : { 'start path': source.startPath || undefined }
-      const sourceInfo = yaml.dump({ url, branches, tags, ...startPathInfo }, { flowLevel: 1 }).trimRight()
+      const sourceInfo = yaml.dump({ url, branches, tags, commits, ...startPathInfo }, { flowLevel: 1 }).trimRight()
       logger.info(`No matching references found for content source entry (${sourceInfo.replace(NEWLINE_RX, ' | ')})`)
     }
   }
@@ -293,7 +298,7 @@ async function selectStartPathsForRepository (repo, sources) {
 
 // QUESTION should we resolve HEAD to a ref eagerly to avoid having to do a match on it?
 async function selectReferences (source, repo, remote) {
-  let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns } = source
+  let { branches: branchPatterns, tags: tagPatterns, commits, worktrees: worktreePatterns } = source
   const managed = 'url' in repo
   const isBare = managed || repo.noCheckout
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
@@ -309,11 +314,20 @@ async function selectReferences (source, repo, remote) {
     const tags = await git.listTags(repo)
     if (tags.length) {
       for (const shortname of filterRefs(tags, tagPatterns, patternCache)) {
-        // NOTE tags are stored using symbol keys to distinguish them from branches
-        refs.set(Symbol(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
+        // NOTE tags are stored using Buffer keys to distinguish them from commits and branches
+        refs.set(Buffer.from(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
       }
     }
   }
+  if (
+    commits &&
+    (commits = Array.isArray(commits) ? commits.map((commit) => String(commit)) : commits.split(CSV_RX)).length
+  ) {
+    for (const oid of commits) {
+      // NOTE commits are stored using Symbol keys to distinguish them from tags and branches
+      refs.set(Symbol(oid), { oid, shortname: oid, fullname: 'commits/' + oid, type: 'commit' })
+    }
+  }
   if (
     !branchPatterns ||
     !(branchPatterns = Array.isArray(branchPatterns)
@@ -402,9 +416,11 @@ async function selectReferences (source, repo, remote) {
       if (currentBranch != null) return [currentBranch]
       return getCurrentBranchName(repo).then((branch) => (branch ? [branch] : []))
     })
-    const worktrees = await findWorktrees(repo, worktreePatterns, useWorktree)
-    let onMatch
-    if ((useWorktree || worktreePatterns.length) && worktrees.size) {
+    let onMatch, worktrees
+    if (
+      (useWorktree || worktreePatterns.length) &&
+      (worktrees = await findWorktrees(repo, worktreePatterns, useWorktree)).size
+    ) {
       const headNames = new Map()
       worktrees.forEach(({ name, symbolicNames }, shortname) => {
         if (name) {
@@ -430,6 +446,7 @@ async function selectReferences (source, repo, remote) {
       const preferRemote = isBare && remoteBranches.length > 0
       for (const shortname of filterRefs(localBranches, branchPatterns, patternCache, onMatch)) {
         if (preferRemote && refs.has(shortname)) continue
+        worktrees ??= await findWorktrees(repo, worktreePatterns, useWorktree)
         const head = (worktrees.get(shortname) || { head: false }).head
         refs.set(shortname, { shortname, fullname: 'heads/' + shortname, type: 'branch', head })
       }
@@ -456,9 +473,11 @@ async function selectStartPaths (source, repo, ref) {
   const displayUrl = url || repo.dir
   const worktreePath = ref.head
   if (!worktreePath) {
-    ref.oid = await git.resolveRef(
-      Object.assign(ref.detached ? { ref: 'HEAD', depth: 1 } : { ref: 'refs/' + ref.fullname }, repo)
-    )
+    ref.oid = ref.oid
+      ? await git.expandOid(Object.assign({ oid: ref.oid }, repo)).catch(() => ref.oid)
+      : await git.resolveRef(
+          Object.assign(ref.detached ? { ref: 'HEAD', depth: 1 } : { ref: 'refs/' + ref.fullname }, repo)
+        )
   }
   if ('startPaths' in source) {
     let startPaths
@@ -493,7 +512,9 @@ function collectFilesFromStartPath (startPath, repo, authStatus, ref, originUrl,
   return (worktreePath ? readFilesFromWorktree(origin) : readFilesFromGitTree(repo, ref.oid, startPath))
     .then((files) => {
       const batch = deepClone((origin.descriptor = loadComponentDescriptor(files, ref, version)))
-      if ('nav' in batch && Array.isArray(batch.nav)) batch.nav.origin = origin
+      if ('nav' in batch && Array.isArray(batch.nav)) {
+        Object.defineProperty(batch.nav, 'origin', { configurable: true, value: origin, writable: true })
+      }
       batch.files = files.map((file) => assignFileProperties(file, origin))
       batch.origins = [origin]
       return batch
@@ -825,6 +846,7 @@ function buildFetchOptions (repo, progress, displayUrl, credentialsFromUrl, gitP
   } else if (!fetch.tags) {
     opts.noTags = true
   }
+  if (fetch.commits) opts.oids = fetch.commits
   return opts
 }
 
@@ -859,7 +881,10 @@ function createProgressListener (progress, progressLabel, operation) {
   // NOTE leave room for indeterminate progress at end of bar; this isn't strictly needed for a bare clone
   progressBar.scaleFactor = Math.max(0, (ticks - 1) / ticks)
   progressBar.tick(0)
-  return Object.assign(onGitProgress.bind(progressBar), { finish: onGitComplete.bind(progressBar) })
+  return Object.assign(onGitProgress.bind(progressBar), {
+    finish: onGitComplete.bind(progressBar),
+    reset: () => progressBar.update(0),
+  })
 }
 
 function formatProgressBar (label, maxLabelWidth, operation) {
@@ -975,6 +1000,16 @@ function tagsSpecified (sources) {
   return sources.some(({ tags }) => tags && (Array.isArray(tags) ? tags.length : true))
 }
 
+function commitsRequested (sources) {
+  if (!sources.some(({ commits }) => commits && (Array.isArray(commits) ? commits.length : true))) return
+  const result = new Set()
+  for (const { commits } of sources) {
+    if (!commits) continue
+    for (const commit of Array.isArray(commits) ? commits : commits.split(CSV_RX)) result.add(String(commit))
+  }
+  return [...result]
+}
+
 function loadGitPlugins (gitConfig, networkConfig, startDir) {
   const plugins = new Map((git.cores || git.default.cores || new Map()).get(GIT_CORE))
   for (const [name, request] of Object.entries(gitConfig.plugins || {})) {
@@ -1164,4 +1199,27 @@ async function promiseAllWithLimit (tasks, limit = Infinity) {
   return Promise.all(started)
 }
 
+async function ensureOids (opts) {
+  if (!opts.oids) return
+  let prevShallowCommits = await getShallowCommits(opts)
+  if (prevShallowCommits == null) return
+  let oids = opts.oids.slice()
+  const deepenOpts = Object.assign({}, opts, { relative: true })
+  const format = 'deflated'
+  while (oids.length) {
+    deepenOpts.onProgress?.reset()
+    await git.fetch(deepenOpts)
+    const shallowCommits = await getShallowCommits(opts)
+    if (shallowCommits == null || shallowCommits === prevShallowCommits) break
+    prevShallowCommits = shallowCommits
+    oids = await Promise.all(
+      oids.map((oid) => git.readObject(Object.assign({ oid, format }, opts)).then(invariably.void, () => oid))
+    ).then((results) => results.filter((it) => it))
+  }
+}
+
+function getShallowCommits ({ gitdir }) {
+  return fsp.readFile(ospath.join(gitdir, 'shallow'), 'utf8').catch(invariably.void)
+}
+
 module.exports = aggregateContent

package/lib/compute-origin.js

@@ -42,7 +42,7 @@ function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath =
   } else if (editUrl) {
     const vars = {
       path: () => (startPath ? path.join(startPath, '%s') : '%s'),
-      ref: () => 'refs/' + (reftype === 'branch' ? 'heads' : reftype) + '/' + refname,
+      ref: () => 'refs/' + (reftype === 'branch' ? 'head' : reftype) + 's/' + refname,
       refhash: () => refhash,
       reftype: () => reftype,
       refname: () => refname,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.2.0-alpha.10",
+  "version": "3.2.0-alpha.12",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -33,13 +33,13 @@
   },
   "dependencies": {
     "@antora/expand-path-helper": "~3.0",
-    "@antora/logger": "3.2.0-alpha.10",
+    "@antora/logger": "3.2.0-alpha.12",
     "@antora/user-require-helper": "~3.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
     "fast-glob": "~3.3",
     "hpagent": "~1.2",
-    "isomorphic-git": "~1.25",
+    "isomorphic-git": "~1.37",
     "js-yaml": "~4.1",
     "multi-progress": "~4.0",
     "picomatch": "~4.0",
@@ -65,7 +65,7 @@
     "web publishing"
   ],
   "scripts": {
-    "test": "_mocha",
+    "test": "node --test",
     "prepublishOnly": "npx -y downdoc@latest --prepublish",
     "postpublish": "npx -y downdoc@latest --postpublish"
   }