@antora/content-aggregator 3.2.0-alpha.10 → 3.2.0-alpha.11

package/lib/aggregate-content.js

@@ -116,6 +116,8 @@ async function collectFiles (sourcesByUrl, loadOpts, concurrency, fetchedUrls =
     const loadOptsForUrl = Object.assign({}, loadOpts)
     if (loadOpts.fetch.always && fetchedUrls.length && ~fetchedUrls.indexOf(url)) loadOptsForUrl.fetch.always = false
     if (tagsSpecified(sources)) loadOptsForUrl.fetch.tags = true
+    const commits = commitsRequested(sources)
+    if (commits) loadOptsForUrl.fetch.commits = commits
     return loadRepository.bind(null, url, loadOptsForUrl, { url, sources })
   })
   return gracefulPromiseAllWithLimit(loadTasks, concurrency.fetch).then(([results, rejections]) => {
@@ -177,6 +179,7 @@ async function loadRepository (url, opts, result = {}) {
         const fetchOpts = buildFetchOptions(repo, progress, displayUrl, credentials, gitPlugins, fetch, 'fetch')
         await git
           .fetch(fetchOpts)
+          .then(() => ensureOids(fetchOpts))
           .then(() => {
             authStatus = identifyAuthStatus(credentialManager, credentials, url)
             return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
@@ -198,6 +201,7 @@ async function loadRepository (url, opts, result = {}) {
       await git
         .clone(fetchOpts)
         .then(() => git.resolveRef(Object.assign({ ref: 'HEAD', depth: 1 }, repo)))
+        .then(() => ensureOids(fetchOpts))
         .then(() => {
           authStatus = identifyAuthStatus(credentialManager, credentials, url)
           return git.setConfig(Object.assign({ path: 'remote.origin.private', value: authStatus }, repo))
@@ -279,12 +283,12 @@ async function selectStartPathsForRepository (repo, sources) {
         }
       }
     } else {
-      const { url, branches, tags } = source
+      const { url, branches, tags, commits } = source
       const startPathInfo =
         'startPaths' in source
           ? { 'start paths': source.startPaths || undefined }
           : { 'start path': source.startPath || undefined }
-      const sourceInfo = yaml.dump({ url, branches, tags, ...startPathInfo }, { flowLevel: 1 }).trimRight()
+      const sourceInfo = yaml.dump({ url, branches, tags, commits, ...startPathInfo }, { flowLevel: 1 }).trimRight()
       logger.info(`No matching references found for content source entry (${sourceInfo.replace(NEWLINE_RX, ' | ')})`)
     }
   }
@@ -293,7 +297,7 @@ async function selectStartPathsForRepository (repo, sources) {
 
 // QUESTION should we resolve HEAD to a ref eagerly to avoid having to do a match on it?
 async function selectReferences (source, repo, remote) {
-  let { branches: branchPatterns, tags: tagPatterns, worktrees: worktreePatterns } = source
+  let { branches: branchPatterns, tags: tagPatterns, commits, worktrees: worktreePatterns } = source
   const managed = 'url' in repo
   const isBare = managed || repo.noCheckout
   const patternCache = repo.cache[REF_PATTERN_CACHE_KEY]
@@ -309,11 +313,20 @@ async function selectReferences (source, repo, remote) {
     const tags = await git.listTags(repo)
     if (tags.length) {
       for (const shortname of filterRefs(tags, tagPatterns, patternCache)) {
-        // NOTE tags are stored using symbol keys to distinguish them from branches
-        refs.set(Symbol(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
+        // NOTE tags are stored using Buffer keys to distinguish them from commits and branches
+        refs.set(Buffer.from(shortname), { shortname, fullname: 'tags/' + shortname, type: 'tag', head: noWorktree })
       }
     }
   }
+  if (
+    commits &&
+    (commits = Array.isArray(commits) ? commits.map((commit) => String(commit)) : commits.split(CSV_RX)).length
+  ) {
+    for (const oid of commits) {
+      // NOTE commits are stored using Symbol keys to distinguish them from tags and branches
+      refs.set(Symbol(oid), { oid, shortname: oid, fullname: 'commits/' + oid, type: 'commit' })
+    }
+  }
   if (
     !branchPatterns ||
     !(branchPatterns = Array.isArray(branchPatterns)
@@ -456,9 +469,11 @@ async function selectStartPaths (source, repo, ref) {
   const displayUrl = url || repo.dir
   const worktreePath = ref.head
   if (!worktreePath) {
-    ref.oid = await git.resolveRef(
-      Object.assign(ref.detached ? { ref: 'HEAD', depth: 1 } : { ref: 'refs/' + ref.fullname }, repo)
-    )
+    ref.oid = ref.oid
+      ? await git.expandOid(Object.assign({ oid: ref.oid }, repo)).catch(() => ref.oid)
+      : await git.resolveRef(
+          Object.assign(ref.detached ? { ref: 'HEAD', depth: 1 } : { ref: 'refs/' + ref.fullname }, repo)
+        )
   }
   if ('startPaths' in source) {
     let startPaths
@@ -825,6 +840,7 @@ function buildFetchOptions (repo, progress, displayUrl, credentialsFromUrl, gitP
   } else if (!fetch.tags) {
     opts.noTags = true
   }
+  if (fetch.commits) opts.oids = fetch.commits
   return opts
 }
 
@@ -859,7 +875,10 @@ function createProgressListener (progress, progressLabel, operation) {
   // NOTE leave room for indeterminate progress at end of bar; this isn't strictly needed for a bare clone
   progressBar.scaleFactor = Math.max(0, (ticks - 1) / ticks)
   progressBar.tick(0)
-  return Object.assign(onGitProgress.bind(progressBar), { finish: onGitComplete.bind(progressBar) })
+  return Object.assign(onGitProgress.bind(progressBar), {
+    finish: onGitComplete.bind(progressBar),
+    reset: () => progressBar.update(0),
+  })
 }
 
 function formatProgressBar (label, maxLabelWidth, operation) {
@@ -975,6 +994,16 @@ function tagsSpecified (sources) {
   return sources.some(({ tags }) => tags && (Array.isArray(tags) ? tags.length : true))
 }
 
+function commitsRequested (sources) {
+  if (!sources.some(({ commits }) => commits && (Array.isArray(commits) ? commits.length : true))) return
+  const result = new Set()
+  for (const { commits } of sources) {
+    if (!commits) continue
+    for (const commit of Array.isArray(commits) ? commits : commits.split(CSV_RX)) result.add(String(commit))
+  }
+  return [...result]
+}
+
 function loadGitPlugins (gitConfig, networkConfig, startDir) {
   const plugins = new Map((git.cores || git.default.cores || new Map()).get(GIT_CORE))
   for (const [name, request] of Object.entries(gitConfig.plugins || {})) {
@@ -1164,4 +1193,27 @@ async function promiseAllWithLimit (tasks, limit = Infinity) {
   return Promise.all(started)
 }
 
+async function ensureOids (opts) {
+  if (!opts.oids) return
+  let prevShallowCommits = await getShallowCommits(opts)
+  if (prevShallowCommits == null) return
+  let oids = opts.oids.slice()
+  const deepenOpts = Object.assign({}, opts, { relative: true })
+  const format = 'deflated'
+  while (oids.length) {
+    deepenOpts.onProgress?.reset()
+    await git.fetch(deepenOpts)
+    const shallowCommits = await getShallowCommits(opts)
+    if (shallowCommits == null || shallowCommits === prevShallowCommits) break
+    prevShallowCommits = shallowCommits
+    oids = await Promise.all(
+      oids.map((oid) => git.readObject(Object.assign({ oid, format }, opts)).then(invariably.void, () => oid))
+    ).then((results) => results.filter((it) => it))
+  }
+}
+
+function getShallowCommits ({ gitdir }) {
+  return fsp.readFile(ospath.join(gitdir, 'shallow'), 'utf8').catch(invariably.void)
+}
+
 module.exports = aggregateContent

package/lib/compute-origin.js

@@ -42,7 +42,7 @@ function computeOrigin (url, authStatus, gitdir, ref, startPath, worktreePath =
   } else if (editUrl) {
     const vars = {
       path: () => (startPath ? path.join(startPath, '%s') : '%s'),
-      ref: () => 'refs/' + (reftype === 'branch' ? 'heads' : reftype) + '/' + refname,
+      ref: () => 'refs/' + (reftype === 'branch' ? 'head' : reftype) + 's/' + refname,
       refhash: () => refhash,
       reftype: () => reftype,
       refname: () => refname,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.2.0-alpha.10",
+  "version": "3.2.0-alpha.11",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -33,7 +33,7 @@
   },
   "dependencies": {
     "@antora/expand-path-helper": "~3.0",
-    "@antora/logger": "3.2.0-alpha.10",
+    "@antora/logger": "3.2.0-alpha.11",
     "@antora/user-require-helper": "~3.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",