@antora/ui-loader 3.0.0-alpha.9 → 3.0.0-beta.4

package/README.md

@@ -4,7 +4,7 @@ The UI Loader is a component in Antora responsible for downloading and caching t
 The UI files are combined with the content files at the end of the Antora documentation pipeline to produce a complete site.
 
 [Antora](https://antora.org) is a modular static site generator designed for creating documentation sites from AsciiDoc documents.
-Its site generator pipeline aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
+Its site generator aggregates documents from versioned content repositories and processes them using [Asciidoctor](https://asciidoctor.org).
 
 ## Copyright and License
 

package/lib/constants.js

@@ -11,7 +11,7 @@ module.exports = Object.freeze({
     nomount: true,
     nosort: true,
     nounique: true,
-    removeBOM: false,
+    strict: false,
     uniqueBy: (m) => m,
   },
 })

package/lib/file.js

@@ -40,7 +40,7 @@ class ReadableFile extends Readable {
 
   _read () {
     this.push(this._file)
-    this._file = null
+    this.push((this._file = null))
   }
 }
 

package/lib/load-ui.js

@@ -1,24 +1,33 @@
 'use strict'
 
+const { compile: bracesToGroup } = require('braces')
 const camelCaseKeys = require('camelcase-keys')
-const concat = require('simple-concat')
 const { createHash } = require('crypto')
 const expandPath = require('@antora/expand-path-helper')
 const { File, MemoryFile, ReadableFile } = require('./file')
 const { promises: fsp } = require('fs')
 const { concat: get } = require('simple-get')
 const getCacheDir = require('cache-directory')
-const minimatchAll = require('minimatch-all')
+const globStream = require('glob-stream')
 const ospath = require('path')
 const { posix: path } = ospath
+const picomatch = require('picomatch')
 const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
-const { Transform } = require('stream')
-const map = (transform, flush = undefined) => new Transform({ objectMode: true, transform, flush })
+const { pipeline, Transform, Writable } = require('stream')
+const forEach = (write, final) => new Writable({ objectMode: true, write, final })
+const map = (transform) => new Transform({ objectMode: true, transform })
 const UiCatalog = require('./ui-catalog')
 const yaml = require('js-yaml')
-const vfs = require('vinyl-fs')
 const vzip = require('gulp-vinyl-zip')
 
+const STATIC_FILE_MATCHER_OPTS = {
+  expandRange: (begin, end, step, opts) => bracesToGroup(opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`),
+  fastpaths: false,
+  nobracket: true,
+  noquantifiers: true,
+  regex: false,
+  strictSlashes: true,
+}
 const { UI_CACHE_FOLDER, UI_DESC_FILENAME, UI_SRC_GLOB, UI_SRC_OPTS } = require('./constants')
 const URI_SCHEME_RX = /^https?:\/\//
 const EXT_RX = /\.[a-z]{2,3}$/
@@ -31,7 +40,7 @@ const EXT_RX = /\.[a-z]{2,3}$/
  * the playbook. If the path is a URI, it downloads the file and caches it at a
  * unique path to avoid this step in future calls. It then reads all the files
  * from the bundle into memory, skipping any files that fall outside of the
- * start path specified in the ui.startPath property of the playbook.  Finally,
+ * start path specified in the ui.startPath property of the playbook. Finally,
  * it classifies the files and adds them to a UiCatalog, which is then
  * returned.
  *
@@ -48,7 +57,7 @@ const EXT_RX = /\.[a-z]{2,3}$/
  * @param {String} [playbook.network.noProxy=undefined] - The list of domains and IPs that should not be proxied.
  * @param {Object} playbook.ui - The UI configuration object for Antora.
  * @param {String} playbook.ui.bundle - The UI bundle configuration.
- * @param {String} playbook.ui.bundle.url - The path (relative or absolute) or URI
+ * @param {String} playbook.ui.bundle.url - The path (relative or absolute) or URL
  * of the UI bundle to use.
  * @param {String} [playbook.ui.bundle.startPath=''] - The path inside the bundle from
  * which to start reading files.
@@ -94,11 +103,7 @@ async function loadUi (playbook) {
     resolveBundle.then((bundleFile) =>
       new Promise((resolve, reject) =>
         bundleFile.isDirectory()
-          ? vfs
-            .src(UI_SRC_GLOB, Object.assign({ cwd: bundleFile.path }, UI_SRC_OPTS))
-            .on('error', reject)
-            .pipe(relativizeFiles())
-            .pipe(collectFiles(resolve))
+          ? srcFs(ospath.join(bundleFile.path, bundle.startPath || '', '.')).then(resolve, reject)
           : vzip
             .src(bundleFile.path)
             .on('error', (err) => reject(Object.assign(err, { message: `not a valid zip file; ${err.message}` })))
@@ -181,12 +186,12 @@ function downloadBundle (url, to, agent) {
         .on('error', (err) =>
           reject(Object.assign(err, { message: `not a valid zip file; ${err.message}`, summary: 'Invalid UI bundle' }))
         )
-        .on('finish', function () {
+        .on('finish', () =>
           fsp
             .mkdir(ospath.dirname(to), { recursive: true })
             .then(() => fsp.writeFile(to, contents))
             .then(() => resolve(new File({ path: to, stat: { isDirectory: () => false } })))
-        })
+        )
     })
   }).catch((err) => {
     const wrapped = new Error(`${err.summary || 'Failed to download UI bundle'}: ${url}`)
@@ -229,25 +234,25 @@ function bufferizeContents () {
   return map((file, _, next) => {
     // NOTE gulp-vinyl-zip automatically converts the contents of an empty file to a Buffer
     if (file.isStream()) {
-      concat(file.contents, (err, contents) => {
-        if (err) return next(err)
-        file.contents = contents
-        next(null, file)
-      })
+      const buffer = []
+      pipeline(
+        file.contents,
+        forEach((chunk, _, done) => buffer.push(chunk) && done()),
+        (err) => (err ? next(err) : next(null, Object.assign(file, { contents: Buffer.concat(buffer) })))
+      )
     } else {
       next(null, file)
     }
   })
 }
 
-function collectFiles (done) {
-  const files = new Map()
-  return map(
-    (file, _, next) => {
+function collectFiles (resolve, files = new Map()) {
+  return forEach(
+    (file, _, done) => {
       files.set(file.path, file)
-      next()
+      done()
     },
-    () => done(files)
+    (done) => done() || resolve(files)
   )
 }
 
@@ -278,36 +283,16 @@ function srcSupplementalFiles (filesSpec, startDir) {
     ).then((files) => files.reduce((accum, file) => accum.set(file.path, file) && accum, new Map()))
   } else {
     const cwd = expandPath(filesSpec, { dot: startDir })
-    return fsp
-      .access(cwd)
-      .then(
-        () =>
-          new Promise((resolve, reject) =>
-            vfs
-              .src(UI_SRC_GLOB, Object.assign({ cwd, dot: true }, UI_SRC_OPTS))
-              .on('error', reject)
-              .pipe(relativizeFiles())
-              .pipe(collectFiles(resolve))
-          )
-      )
-      .catch((err) => {
+    return fsp.access(cwd).then(
+      () => srcFs(cwd),
+      (err) => {
         // Q: should we skip unreadable files?
         throw Object.assign(err, { message: `problem encountered while reading ui.supplemental_files: ${err.message}` })
-      })
+      }
+    )
   }
 }
 
-function relativizeFiles () {
-  return map((file, _, next) => {
-    if (file.isNull()) {
-      next()
-    } else {
-      const path_ = posixify ? posixify(file.relative) : file.relative
-      next(null, new File({ cwd: file.cwd, path: path_, contents: file.contents, stat: file.stat, local: true }))
-    }
-  })
-}
-
 function mergeFiles (files, supplementalFiles) {
   if (supplementalFiles.size) supplementalFiles.forEach((file) => files.set(file.path, file))
   return files
@@ -318,15 +303,9 @@ function loadConfig (files, outputDir) {
   if (configFile) {
     files.delete(UI_DESC_FILENAME)
     const config = camelCaseKeys(yaml.load(configFile.contents.toString()), { deep: true })
-    if (outputDir !== undefined) config.outputDir = outputDir
     const staticFiles = config.staticFiles
-    if (staticFiles) {
-      if (!Array.isArray(staticFiles)) {
-        config.staticFiles = [staticFiles]
-      } else if (staticFiles.length === 0) {
-        delete config.staticFiles
-      }
-    }
+    if (staticFiles && staticFiles.length) config.isStaticFile = picomatch(staticFiles, STATIC_FILE_MATCHER_OPTS)
+    if (outputDir !== undefined) config.outputDir = outputDir
     return config
   } else {
     return { outputDir }
@@ -334,7 +313,7 @@ function loadConfig (files, outputDir) {
 }
 
 function classifyFile (file, config) {
-  if (config.staticFiles && isStaticFile(file, config.staticFiles)) {
+  if (config.isStaticFile && config.isStaticFile(file.path)) {
     file.type = 'static'
     file.out = resolveOut(file, '')
   } else if (file.isDot()) {
@@ -345,10 +324,6 @@ function classifyFile (file, config) {
   return file
 }
 
-function isStaticFile (file, staticFiles) {
-  return minimatchAll(file.path, staticFiles)
-}
-
 function resolveType (file) {
   const firstPathSegment = file.path.split('/', 1)[0]
   if (firstPathSegment === 'layouts') return 'layout'
@@ -364,4 +339,54 @@ function resolveOut (file, outputDir = '_') {
   return { dirname, basename, path: path.join(dirname, basename) }
 }
 
+function srcFs (cwd) {
+  const relpathStart = cwd.length + 1
+  return new Promise((resolve, reject, cache = Object.create(null), files = new Map()) =>
+    pipeline(
+      globStream(UI_SRC_GLOB, Object.assign({ cache, cwd }, UI_SRC_OPTS)),
+      forEach(({ path: abspathPosix }, _, done) => {
+        if (Array.isArray(cache[abspathPosix])) return done() // detects some directories, but not all
+        const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
+        const relpath = abspath.substr(relpathStart)
+        symlinkAwareStat(abspath).then(
+          (stat) => {
+            if (stat.isDirectory()) return done() // detects remaining directories
+            fsp.readFile(abspath).then(
+              (contents) => {
+                const path_ = posixify ? posixify(relpath) : relpath
+                files.set(path_, new File({ cwd, path: path_, contents, stat, local: true }))
+                done()
+              },
+              (readErr) => {
+                done(Object.assign(readErr, { message: readErr.message.replace(`'${abspath}'`, relpath) }))
+              }
+            )
+          },
+          (statErr) => {
+            if (statErr.symlink) {
+              statErr.message =
+                statErr.code === 'ELOOP'
+                  ? `Symbolic link cycle detected at ${relpath}`
+                  : `Broken symbolic link detected at ${relpath}`
+            } else {
+              statErr.message = statErr.message.replace(`'${abspath}'`, relpath)
+            }
+            done(statErr)
+          }
+        )
+      }),
+      (err) => (err ? reject(err) : resolve(files))
+    )
+  )
+}
+
+function symlinkAwareStat (path_) {
+  return fsp.lstat(path_).then((lstat) => {
+    if (!lstat.isSymbolicLink()) return lstat
+    return fsp.stat(path_).catch((statErr) => {
+      throw Object.assign(statErr, { symlink: true })
+    })
+  })
+}
+
 module.exports = loadUi

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/ui-loader",
-  "version": "3.0.0-alpha.9",
+  "version": "3.0.0-beta.4",
   "description": "Downloads a UI bundle, if necessary, and loads the files into a UI catalog for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -18,20 +18,20 @@
   "main": "lib/index.js",
   "dependencies": {
     "@antora/expand-path-helper": "~2.0",
+    "braces": "~3.0",
     "cache-directory": "~2.0",
-    "camelcase-keys": "~6.2",
+    "camelcase-keys": "~7.0",
+    "glob-stream": "~7.0",
     "gulp-vinyl-zip": "~2.5",
     "hpagent": "~0.1.0",
     "js-yaml": "~4.1",
-    "minimatch-all": "~1.1",
+    "picomatch": "~2.3",
     "should-proxy": "~1.0",
-    "simple-concat": "~1.0",
     "simple-get": "~4.0",
-    "vinyl": "~2.2",
-    "vinyl-fs": "~3.0"
+    "vinyl": "~2.2"
   },
   "engines": {
-    "node": ">=10.17.0"
+    "node": ">=12.21.0"
   },
   "files": [
     "lib/"
@@ -45,5 +45,6 @@
     "static site",
     "web publishing"
   ],
-  "gitHead": "a504d6889819b548e8a5416a7194cbb6f9a93e93"
+  "gitHead": "8a142499e9f1a9e0631777796e06dd6c010d3a90",
+  "readmeFilename": "README.md"
 }