@antora/ui-loader 3.0.0-alpha.8 → 3.0.0-beta.3

package/README.md

@@ -4,7 +4,7 @@ The UI Loader is a component in Antora responsible for downloading and caching t
 The UI files are combined with the content files at the end of the Antora documentation pipeline to produce a complete site.
 
 [Antora](https://antora.org) is a modular static site generator designed for creating documentation sites from AsciiDoc documents.
-Its site generator pipeline aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
+Its site generator aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
 
 ## Copyright and License
 

package/lib/constants.js

@@ -11,7 +11,7 @@ module.exports = Object.freeze({
     nomount: true,
     nosort: true,
     nounique: true,
-    removeBOM: false,
+    strict: false,
     uniqueBy: (m) => m,
   },
 })

package/lib/load-ui.js

@@ -1,5 +1,6 @@
 'use strict'
 
+const { compile: bracesToGroup } = require('braces')
 const camelCaseKeys = require('camelcase-keys')
 const concat = require('simple-concat')
 const { createHash } = require('crypto')
@@ -8,17 +9,25 @@ const { File, MemoryFile, ReadableFile } = require('./file')
 const { promises: fsp } = require('fs')
 const { concat: get } = require('simple-get')
 const getCacheDir = require('cache-directory')
-const minimatchAll = require('minimatch-all')
+const globStream = require('glob-stream')
 const ospath = require('path')
 const { posix: path } = ospath
+const picomatch = require('picomatch')
 const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
-const { Transform } = require('stream')
+const { pipeline, Transform } = require('stream')
 const map = (transform, flush = undefined) => new Transform({ objectMode: true, transform, flush })
 const UiCatalog = require('./ui-catalog')
 const yaml = require('js-yaml')
-const vfs = require('vinyl-fs')
 const vzip = require('gulp-vinyl-zip')
 
+const STATIC_FILE_MATCHER_OPTS = {
+  expandRange: (begin, end, step, opts) => bracesToGroup(opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`),
+  fastpaths: false,
+  nobracket: true,
+  noquantifiers: true,
+  regex: false,
+  strictSlashes: true,
+}
 const { UI_CACHE_FOLDER, UI_DESC_FILENAME, UI_SRC_GLOB, UI_SRC_OPTS } = require('./constants')
 const URI_SCHEME_RX = /^https?:\/\//
 const EXT_RX = /\.[a-z]{2,3}$/
@@ -48,7 +57,7 @@ const EXT_RX = /\.[a-z]{2,3}$/
  * @param {String} [playbook.network.noProxy=undefined] - The list of domains and IPs that should not be proxied.
  * @param {Object} playbook.ui - The UI configuration object for Antora.
  * @param {String} playbook.ui.bundle - The UI bundle configuration.
- * @param {String} playbook.ui.bundle.url - The path (relative or absolute) or URI
+ * @param {String} playbook.ui.bundle.url - The path (relative or absolute) or URL
  * of the UI bundle to use.
  * @param {String} [playbook.ui.bundle.startPath=''] - The path inside the bundle from
  * which to start reading files.
@@ -73,32 +82,28 @@ async function loadUi (playbook) {
       const cachePath = ospath.join(absCacheDir, `${sha1(bundleUrl)}.zip`)
       return fetch && bundle.snapshot
         ? downloadBundle(bundleUrl, cachePath, createAgent(bundleUrl, playbook.network || {}))
-        : fsp
-          .stat(cachePath)
-          .then((stat) => new File({ path: cachePath, stat }))
-          .catch(() => downloadBundle(bundleUrl, cachePath, createAgent(bundleUrl, playbook.network || {})))
+        : fsp.stat(cachePath).then(
+          (stat) => new File({ path: cachePath, stat }),
+          () => downloadBundle(bundleUrl, cachePath, createAgent(bundleUrl, playbook.network || {}))
+        )
     })
   } else {
     const localPath = expandPath(bundleUrl, { dot: startDir })
-    resolveBundle = fsp
-      .stat(localPath)
-      .then((stat) => new File({ path: localPath, stat }))
-      .catch(() => {
+    resolveBundle = fsp.stat(localPath).then(
+      (stat) => new File({ path: localPath, stat }),
+      () => {
         throw new Error(
           `Specified UI ${path.extname(localPath) ? 'bundle' : 'directory'} does not exist: ` +
             (bundleUrl === localPath ? bundleUrl : `${localPath} (resolved from url: ${bundleUrl})`)
         )
-      })
+      }
+    )
   }
   const files = await Promise.all([
     resolveBundle.then((bundleFile) =>
       new Promise((resolve, reject) =>
         bundleFile.isDirectory()
-          ? vfs
-            .src(UI_SRC_GLOB, Object.assign({ cwd: bundleFile.path }, UI_SRC_OPTS))
-            .on('error', reject)
-            .pipe(relativizeFiles())
-            .pipe(collectFiles(resolve))
+          ? srcFs(bundleFile.path).then(resolve, reject)
           : vzip
             .src(bundleFile.path)
             .on('error', (err) => reject(Object.assign(err, { message: `not a valid zip file; ${err.message}` })))
@@ -149,16 +154,16 @@ function ensureCacheDir (customCacheDir, startDir) {
       ? getCacheDir('antora' + (process.env.NODE_ENV === 'test' ? '-test' : '')) || ospath.resolve('.antora/cache')
       : expandPath(customCacheDir, { dot: startDir })
   const cacheDir = ospath.join(baseCacheDir, UI_CACHE_FOLDER)
-  return fsp
-    .mkdir(cacheDir, { recursive: true })
-    .then(() => cacheDir)
-    .catch((err) => {
+  return fsp.mkdir(cacheDir, { recursive: true }).then(
+    () => cacheDir,
+    (err) => {
       throw Object.assign(err, { message: `Failed to create UI cache directory: ${cacheDir}; ${err.message}` })
-    })
+    }
+  )
 }
 
 function createAgent (url, { httpProxy, httpsProxy, noProxy }) {
-  if (httpsProxy || httpProxy) {
+  if ((httpsProxy || httpProxy) && noProxy !== '*') {
     const { HttpProxyAgent, HttpsProxyAgent } = require('hpagent')
     const shouldProxy = require('should-proxy')
     const proxy = url.startsWith('https:')
@@ -280,16 +285,7 @@ function srcSupplementalFiles (filesSpec, startDir) {
     const cwd = expandPath(filesSpec, { dot: startDir })
     return fsp
       .access(cwd)
-      .then(
-        () =>
-          new Promise((resolve, reject) =>
-            vfs
-              .src(UI_SRC_GLOB, Object.assign({ cwd, dot: true }, UI_SRC_OPTS))
-              .on('error', reject)
-              .pipe(relativizeFiles())
-              .pipe(collectFiles(resolve))
-          )
-      )
+      .then(() => srcFs(cwd))
       .catch((err) => {
         // Q: should we skip unreadable files?
         throw Object.assign(err, { message: `problem encountered while reading ui.supplemental_files: ${err.message}` })
@@ -297,17 +293,6 @@ function srcSupplementalFiles (filesSpec, startDir) {
   }
 }
 
-function relativizeFiles () {
-  return map((file, _, next) => {
-    if (file.isNull()) {
-      next()
-    } else {
-      const path_ = posixify ? posixify(file.relative) : file.relative
-      next(null, new File({ cwd: file.cwd, path: path_, contents: file.contents, stat: file.stat, local: true }))
-    }
-  })
-}
-
 function mergeFiles (files, supplementalFiles) {
   if (supplementalFiles.size) supplementalFiles.forEach((file) => files.set(file.path, file))
   return files
@@ -318,15 +303,9 @@ function loadConfig (files, outputDir) {
   if (configFile) {
     files.delete(UI_DESC_FILENAME)
     const config = camelCaseKeys(yaml.load(configFile.contents.toString()), { deep: true })
-    if (outputDir !== undefined) config.outputDir = outputDir
     const staticFiles = config.staticFiles
-    if (staticFiles) {
-      if (!Array.isArray(staticFiles)) {
-        config.staticFiles = [staticFiles]
-      } else if (staticFiles.length === 0) {
-        delete config.staticFiles
-      }
-    }
+    if (staticFiles && staticFiles.length) config.isStaticFile = picomatch(staticFiles, STATIC_FILE_MATCHER_OPTS)
+    if (outputDir !== undefined) config.outputDir = outputDir
     return config
   } else {
     return { outputDir }
@@ -334,7 +313,7 @@ function loadConfig (files, outputDir) {
 }
 
 function classifyFile (file, config) {
-  if (config.staticFiles && isStaticFile(file, config.staticFiles)) {
+  if (config.isStaticFile && config.isStaticFile(file.path)) {
     file.type = 'static'
     file.out = resolveOut(file, '')
   } else if (file.isDot()) {
@@ -345,10 +324,6 @@ function classifyFile (file, config) {
   return file
 }
 
-function isStaticFile (file, staticFiles) {
-  return minimatchAll(file.path, staticFiles)
-}
-
 function resolveType (file) {
   const firstPathSegment = file.path.split('/', 1)[0]
   if (firstPathSegment === 'layouts') return 'layout'
@@ -364,4 +339,52 @@ function resolveOut (file, outputDir = '_') {
   return { dirname, basename, path: path.join(dirname, basename) }
 }
 
+function srcFs (cwd) {
+  return new Promise((resolve, reject, cache = {}, files = new Map()) =>
+    pipeline(
+      globStream(UI_SRC_GLOB, Object.assign({ cache, cwd }, UI_SRC_OPTS)),
+      map(({ path: abspathPosix }, _, next) => {
+        const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
+        const relpath = abspath.substr(cwd.length + 1)
+        symlinkAwareStat(abspath).then(
+          (stat) => {
+            if (stat.isDirectory()) return next()
+            fsp.readFile(abspath).then(
+              (contents) => {
+                const path_ = posixify ? posixify(relpath) : relpath
+                files.set(path_, new File({ cwd, path: path_, contents, stat, local: true }))
+                next()
+              },
+              (readErr) => {
+                next(Object.assign(readErr, { message: readErr.message.replace(`'${abspath}'`, relpath) }))
+              }
+            )
+          },
+          (statErr) => {
+            if (statErr.symlink) {
+              statErr.message =
+                statErr.code === 'ELOOP'
+                  ? `Symbolic link cycle detected at ${relpath}`
+                  : `Broken symbolic link detected at ${relpath}`
+            } else {
+              statErr.message = statErr.message.replace(`'${abspath}'`, relpath)
+            }
+            next(statErr)
+          }
+        )
+      }),
+      (err) => (err ? reject(err) : resolve(files))
+    )
+  )
+}
+
+function symlinkAwareStat (path_) {
+  return fsp.lstat(path_).then((lstat) => {
+    if (!lstat.isSymbolicLink()) return lstat
+    return fsp.stat(path_).catch((statErr) => {
+      throw Object.assign(statErr, { symlink: true })
+    })
+  })
+}
+
 module.exports = loadUi

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/ui-loader",
-  "version": "3.0.0-alpha.8",
+  "version": "3.0.0-beta.3",
   "description": "Downloads a UI bundle, if necessary, and loads the files into a UI catalog for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -18,20 +18,20 @@
   "main": "lib/index.js",
   "dependencies": {
     "@antora/expand-path-helper": "~2.0",
+    "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~6.2",
+    "camelcase-keys": "~7.0",
     "gulp-vinyl-zip": "~2.5",
     "hpagent": "~0.1.0",
     "js-yaml": "~4.1",
-    "minimatch-all": "~1.1",
+    "picomatch": "~2.3",
     "should-proxy": "~1.0",
     "simple-concat": "~1.0",
     "simple-get": "~4.0",
-    "vinyl": "~2.2",
-    "vinyl-fs": "~3.0"
+    "vinyl": "~2.2"
   },
   "engines": {
-    "node": ">=10.17.0"
+    "node": ">=12.21.0"
   },
   "files": [
     "lib/"
@@ -45,5 +45,5 @@
     "static site",
     "web publishing"
   ],
-  "gitHead": "2e5695bea11fb5719989c329c97e66d36e29659f"
+  "gitHead": "45da95a2e2dea538379d2d9f42013d2208fb86c3"
 }