@antora/content-aggregator 3.0.0-alpha.6 → 3.0.0-beta.1

package/lib/constants.js

@@ -2,24 +2,25 @@
 
 module.exports = Object.freeze({
   COMPONENT_DESC_FILENAME: 'antora.yml',
-  CONTENT_CACHE_FOLDER: 'content/2',
-  CONTENT_GLOB: '**/*.*',
+  CONTENT_CACHE_FOLDER: 'content',
+  CONTENT_SRC_GLOB: '**/*[!~]',
+  CONTENT_SRC_OPTS: { follow: true, nomount: true, nosort: true, nounique: true, removeBOM: false, uniqueBy: (m) => m },
   FILE_MODES: { 100644: 0o100666 & ~process.umask(), 100755: 0o100777 & ~process.umask() },
   GIT_CORE: 'antora',
   GIT_OPERATION_LABEL_LENGTH: 8,
   GIT_PROGRESS_PHASES: ['Counting objects', 'Compressing objects', 'Receiving objects', 'Resolving deltas'],
   PICOMATCH_VERSION_OPTS: {
     bash: true,
-    debug: false,
     dot: true,
     fastpaths: false,
-    lookbehinds: false,
-    noextglob: true,
+    nobracket: true,
     noglobstar: true,
     nonegate: true,
     noquantifiers: true,
+    regex: false,
     strictSlashes: true,
   },
+  REF_PATTERN_CACHE_KEY: Symbol('RefPatternCache'),
   SYMLINK_FILE_MODE: '120000',
   VALID_STATE_FILENAME: 'valid',
 })

package/lib/decode-uint8-array.js

@@ -0,0 +1,6 @@
+'use strict'
+
+module.exports = (({ StringDecoder }) => {
+  const decoder = new StringDecoder()
+  return decoder.write.bind(decoder)
+})(require('string_decoder'))

package/lib/filter-refs.js

@@ -0,0 +1,60 @@
+'use strict'
+
+const { compile: bracesToGroup } = require('braces')
+const { makeRe: makePicomatchRx } = require('picomatch')
+
+function getPicomatchOpts (cache) {
+  return {
+    bash: true,
+    dot: true,
+    expandRange: (begin, end, step, opts) => {
+      const pattern = opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`
+      return cache.braces.get(pattern) || cache.braces.set(pattern, bracesToGroup(pattern)).get(pattern)
+    },
+    fastpaths: false,
+    nobracket: true,
+    noglobstar: true,
+    noquantifiers: true,
+    regex: false,
+    strictSlashes: true,
+  }
+}
+
+function compileRx (pattern, opts) {
+  if (pattern === '*' || pattern === '**') return { test: () => true }
+  return pattern.charAt() === '!' // do our own negate
+    ? Object.defineProperty(makePicomatchRx(pattern.substr(1), opts), 'negated', { value: true })
+    : makePicomatchRx(pattern, opts)
+}
+
+function createMatcher (patterns, cache) {
+  let opts
+  const rxs = patterns.map(
+    (pattern) =>
+      cache.get(pattern) ||
+      cache.set(pattern, compileRx(pattern, opts || (opts = getPicomatchOpts(cache)))).get(pattern)
+  )
+  return (candidate) => {
+    let first = true
+    let matched
+    for (const rx of rxs) {
+      if (matched) {
+        if (rx.negated && rx.test(candidate)) return
+      } else if (first || !rx.negated) {
+        matched = rx.test(candidate)
+      }
+      first = false
+    }
+    return matched
+  }
+}
+
+function filterRefs (candidates, patterns, cache = Object.assign(new Map(), { braces: new Map() })) {
+  const isMatch = createMatcher(patterns, cache)
+  return candidates.reduce((accum, candidate) => {
+    if (isMatch(candidate)) accum.push(candidate)
+    return accum
+  }, [])
+}
+
+module.exports = filterRefs

package/lib/git-credential-manager-store.js

@@ -2,6 +2,7 @@
 
 const { homedir } = require('os')
 const expandPath = require('@antora/expand-path-helper')
+const invariably = { void: () => undefined }
 const { promises: fsp } = require('fs')
 const ospath = require('path')
 
@@ -9,12 +10,10 @@ class GitCredentialManagerStore {
   configure ({ config, startDir }) {
     this.entries = undefined
     this.urls = {}
-    if ((this.contents = (config = config || {}).contents)) {
+    if ((this.contents = (config = config || {}).contents) || !config.path) {
       this.path = undefined
-    } else if (config.path) {
-      this.path = expandPath(config.path, '~+', startDir)
     } else {
-      this.path = undefined
+      this.path = expandPath(config.path, { dot: startDir })
     }
     return this
   }
@@ -28,10 +27,7 @@ class GitCredentialManagerStore {
         contentsPromise = Promise.resolve(this.contents)
         delimiter = /[,\n]/
       } else if (this.path) {
-        contentsPromise = fsp
-          .access(this.path)
-          .then(() => fsp.readFile(this.path, 'utf8'))
-          .catch(() => undefined)
+        contentsPromise = fsp.access(this.path).then(() => fsp.readFile(this.path, 'utf8'), invariably.void)
       } else {
         const homeGitCredentialsPath = ospath.join(homedir(), '.git-credentials')
         const xdgConfigGitCredentialsPath = ospath.join(
@@ -45,8 +41,7 @@ class GitCredentialManagerStore {
           .catch(() =>
             fsp
               .access(xdgConfigGitCredentialsPath)
-              .then(() => fsp.readFile(xdgConfigGitCredentialsPath, 'utf8'))
-              .catch(() => undefined)
+              .then(() => fsp.readFile(xdgConfigGitCredentialsPath, 'utf8'), invariably.void)
           )
       }
       contentsPromise.then((contents) => {
@@ -62,7 +57,7 @@ class GitCredentialManagerStore {
                   if (password) {
                     credentials = { username: decodeURIComponent(username), password: decodeURIComponent(password) }
                   } else if (username) {
-                    credentials = { token: decodeURIComponent(username) }
+                    credentials = { username: decodeURIComponent(username), password: '' }
                   } else {
                     return accum
                   }
@@ -101,7 +96,7 @@ class GitCredentialManagerStore {
     const statusCode = 401
     const statusMessage = 'HTTP Basic: Access Denied'
     const err = new Error(`HTTP Error: ${statusCode} ${statusMessage}`)
-    err.name = err.code = 'HTTPError'
+    err.name = err.code = 'HttpError'
     err.data = { statusCode, statusMessage }
     if (auth) err.rejected = true
     throw err

package/lib/git-plugin-http.js

@@ -1,32 +1,56 @@
 'use strict'
 
 const get = require('simple-get')
-const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent')
-const shouldProxy = require('should-proxy')
 
-module.exports = ({ httpProxy, httpsProxy, noProxy }) => {
-  return async ({ core, emitter, emitterPreifx, url, method, headers, body }) => {
-    if (body && Array.isArray(body)) {
-      const buffers = []
-      for await (const chunk of body) buffers.push(Buffer.from(chunk))
-      body = Buffer.concat(buffers)
-    }
-    const proxy = url.startsWith('https:')
-      ? { ProxyAgent: HttpsProxyAgent, url: httpsProxy }
-      : { ProxyAgent: HttpProxyAgent, url: httpProxy }
-    let agent
-    if (proxy.url && shouldProxy(url, { no_proxy: noProxy })) {
-      // see https://github.com/delvedor/hpagent/issues/18
-      const { protocol, hostname, port, username, password } = new URL(proxy.url)
-      const proxyUrl = { protocol, hostname, port, username: username || null, password: password || null }
-      agent = new proxy.ProxyAgent({ proxy: proxyUrl })
+function distillResponse (res) {
+  const { url, method, statusCode, statusMessage, headers } = res
+  return { url, method, statusCode, statusMessage, headers, body: res }
+}
+
+async function mergeBuffers (data) {
+  if (!Array.isArray(data)) return data
+  if (data.length === 1 && data[0] instanceof Buffer) return data[0]
+  const buffers = []
+  let offset = 0
+  let size = 0
+  for await (const chunk of data) {
+    buffers.push(chunk)
+    size += chunk.byteLength
+  }
+  data = new Uint8Array(size)
+  for (const buffer of buffers) {
+    data.set(buffer, offset)
+    offset += buffer.byteLength
+  }
+  return Buffer.from(data.buffer)
+}
+
+module.exports = ({ httpProxy, httpsProxy, noProxy }, userAgent) => {
+  if ((httpsProxy || httpProxy) && noProxy !== '*') {
+    const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent')
+    const shouldProxy = require('should-proxy')
+    return {
+      async request ({ url, method, headers, body }) {
+        headers['user-agent'] = userAgent
+        body = await mergeBuffers(body)
+        const proxy = url.startsWith('https:')
+          ? { Agent: HttpsProxyAgent, url: httpsProxy }
+          : { Agent: HttpProxyAgent, url: httpProxy }
+        const agent =
+          proxy.url && shouldProxy(url, { no_proxy: noProxy }) ? new proxy.Agent({ proxy: proxy.url }) : undefined
+        return new Promise((resolve, reject) =>
+          get({ url, method, agent, headers, body }, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
+        )
+      },
     }
-    return new Promise((resolve, reject) =>
-      get({ url, agent, method, headers, body }, (err, res) => {
-        if (err) return reject(err)
-        const { url, method, statusCode, statusMessage, headers } = res
-        resolve({ url, method, statusCode, statusMessage, headers, body: res })
-      })
-    )
+  }
+  return {
+    async request ({ url, method, headers, body }) {
+      headers['user-agent'] = userAgent
+      body = await mergeBuffers(body)
+      return new Promise((resolve, reject) =>
+        get({ url, method, headers, body }, (err, res) => (err ? reject(err) : resolve(distillResponse(res))))
+      )
+    },
   }
 }

package/lib/git.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = require('isomorphic-git')

package/lib/index.js

@@ -5,7 +5,7 @@
  *
  * Responsible for aggregating the content from multiple repositories and
  * references into a raw aggregate of virtual files that can be organized by a
- * subsequent step in the pipeline.
+ * subsequent step in the generator.
  *
  * @namespace content-aggregator
  */

package/lib/promise-all-settled-polyfill.js

@@ -0,0 +1,11 @@
+'use strict'
+
+Promise.allSettled = (iterable) =>
+  Promise.all(
+    iterable.map((it) =>
+      it.then(
+        (value) => ({ status: 'fulfilled', value }),
+        (reason) => ({ status: 'rejected', reason })
+      )
+    )
+  )

package/lib/resolve-path-globs.js

@@ -1,26 +1,33 @@
 'use strict'
 
-const { expand: expandBraces } = require('braces')
+const { expand: expandBraces, compile: bracesToGroup } = require('braces')
 const flattenDeep = require('./flatten-deep')
 const { promises: fsp } = require('fs')
-const git = require('isomorphic-git')
-const invariably = { true: () => true, false: () => false, void: () => {}, emptyArray: () => [] }
+const git = require('./git')
+const invariably = { true: () => true, false: () => false, void: () => undefined, emptyArray: () => [] }
 const { makeRe: makePicomatchRx } = require('picomatch')
 
-const RX_ESCAPE_EXCEPT_GLOB = /[.+?^${}()|[\]\\]/g
-const RX_MAGIC_DETECTOR = /[*{]/
-const RX_QUESTION_MARK = /\?/g
-const PICOMATCH_OPTS = { nobracket: true, noextglob: true, noglobstar: true, noquantifiers: true }
-const PICOMATCH_NEGATED_OPTS = { nobracket: true, noextglob: true, noquantifiers: true }
+const NON_GLOB_SPECIAL_CHARS_RX = /[.+?^${}()|[\]\\]/g
+const RX_MAGIC_DETECTOR = /[*{(]/
+const PICOMATCH_OPTS = {
+  bash: true,
+  expandRange: (begin, end, step, opts) => bracesToGroup(opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`),
+  fastpaths: false,
+  nobracket: true,
+  noglobstar: true,
+  nonegate: true,
+  noquantifiers: true,
+  regex: false,
+  strictSlashes: true,
+}
 
 function resolvePathGlobs (base, patterns, listDirents, retrievePath, tree = { path: '' }) {
   return patterns.reduce((paths, pattern) => {
     if (pattern.charAt() === '!') {
       return paths.then((resolvedPaths) => {
         if (resolvedPaths.length) {
-          if (~pattern.indexOf('?')) pattern = pattern.replace(RX_QUESTION_MARK, '\\?')
-          const rx = makePicomatchRx(pattern, PICOMATCH_NEGATED_OPTS)
-          return resolvedPaths.filter(rx.test.bind(rx))
+          const rx = makePicomatchRx(pattern.substr(1), PICOMATCH_OPTS)
+          return resolvedPaths.filter((it) => !rx.test(it))
         } else {
           return resolvedPaths
         }
@@ -38,14 +45,13 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
   let patternSegment = patternSegments[0]
   patternSegments = patternSegments.slice(1)
   if (RX_MAGIC_DETECTOR.test(patternSegment)) {
-    let isMatch
-    let explicit
+    let isMatch, explicit
     if (patternSegment === '*') {
       isMatch = (it) => it.charAt() !== '.'
+    } else if (~patternSegment.indexOf('(')) {
+      isMatch = (isMatch = makePicomatchRx(patternSegment, PICOMATCH_OPTS)).test.bind(isMatch)
     } else if (~patternSegment.indexOf('{')) {
       if (globbed) {
-        if (patternSegment.charAt() === '!') patternSegment = '\\' + patternSegment
-        if (~patternSegment.indexOf('?')) patternSegment = patternSegment.replace(RX_QUESTION_MARK, '\\?')
         isMatch = (isMatch = makePicomatchRx(patternSegment, PICOMATCH_OPTS)).test.bind(isMatch)
       } else if (~patternSegment.indexOf('*')) {
         const [wildPatterns, literals] = expandBraces(patternSegment).reduce(
@@ -90,22 +96,15 @@ async function glob (base, patternSegments, listDirents, retrievePath, { oid, pa
         })
       } else if ((patternSegment += '/' + patternSegments.join('/')).indexOf('{')) {
         return expandBraces(patternSegment).map((it) => joinPath(path, it))
-      } else {
-        return [joinPath(path, patternSegment)]
       }
+      return [joinPath(path, patternSegment)]
     } else if (globbed) {
       return (await retrievePath(base, { oid, path }, patternSegment)) ? [joinPath(path, patternSegment)] : []
-    } else {
-      return [joinPath(path, patternSegment)]
     }
+    return [joinPath(path, patternSegment)]
   }
 }
 
-function regexpEscapeWithGlob (str) {
-  // we don't escape "-" since it's meaningless in a literal
-  return str.replace(RX_ESCAPE_EXCEPT_GLOB, '\\$&').replace('*', '.*')
-}
-
 function extractMagicBase (patternSegments, base) {
   let nextSegment
   if (patternSegments.length) {
@@ -123,12 +122,11 @@ function listDirentsFs (base, path) {
 
 function listDirentsGit (repo, treeOid) {
   return git
-    .readObject(Object.assign({ oid: treeOid, filepath: '' }, repo))
-    .catch(() => ({ object: {} }))
-    .then(({ object: { entries } }) =>
-      entries
-        ? entries.map(({ type, oid, path: name }) => ({ name, oid, isDirectory: invariably[type === 'tree'] }))
-        : []
+    .readTree(Object.assign({ oid: treeOid, filepath: '' }, repo))
+    .then(
+      ({ tree: entries }) =>
+        entries.map(({ type, oid, path: name }) => ({ name, oid, isDirectory: invariably[type === 'tree'] })),
+      invariably.emptyArray
     )
 }
 
@@ -141,7 +139,13 @@ function makeMatcherRx (pattern) {
 }
 
 function patternToRx (pattern) {
-  return (pattern.charAt() === '.' ? '' : '(?!\\.)') + regexpEscapeWithGlob(pattern)
+  return (
+    (pattern.charAt() === '.' ? '' : '(?!\\.)') +
+    pattern
+      .replace(NON_GLOB_SPECIAL_CHARS_RX, '\\$&')
+      .replace('\\\\*', '\\x2a')
+      .replace('*', '.*?')
+  )
 }
 
 function readdirWithFileTypes (dir) {
@@ -149,10 +153,7 @@ function readdirWithFileTypes (dir) {
 }
 
 function retrievePathFs (base, { path }, subpath) {
-  return fsp
-    .access(base + '/' + joinPath(path, subpath))
-    .then(invariably.true)
-    .catch(invariably.false)
+  return fsp.access(base + '/' + joinPath(path, subpath)).then(invariably.true, invariably.false)
 }
 
 function retrievePathGit (repo, { oid }, filepath) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/content-aggregator",
-  "version": "3.0.0-alpha.6",
+  "version": "3.0.0-beta.1",
   "description": "Fetches and aggregates content from distributed sources for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -17,16 +17,16 @@
   },
   "main": "lib/index.js",
   "dependencies": {
-    "@antora/expand-path-helper": "~1.0",
+    "@antora/expand-path-helper": "~2.0",
+    "@antora/user-require-helper": "~2.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~6.2",
+    "camelcase-keys": "~7.0",
     "hpagent": "~0.1.0",
-    "isomorphic-git": "0.78.5",
+    "isomorphic-git": "~1.10",
     "js-yaml": "~4.1",
-    "matcher": "~4.0",
     "multi-progress": "~4.0",
-    "picomatch": "~2.2",
+    "picomatch": "~2.3",
     "progress": "~2.0",
     "should-proxy": "~1.0",
     "simple-get": "~4.0",
@@ -34,7 +34,7 @@
     "vinyl-fs": "~3.0"
   },
   "engines": {
-    "node": ">=10.17.0"
+    "node": ">=12.21.0"
   },
   "files": [
     "lib/"
@@ -49,5 +49,5 @@
     "static site",
     "web publishing"
   ],
-  "gitHead": "38ec002e88eede3ce5c401a6e226d1a0356945c5"
+  "gitHead": "7c5ef1ea93dd489af533c80a936c736013c41769"
 }