@antora/ui-loader 3.2.0-alpha.1 → 3.2.0-alpha.10

package/lib/constants.js

@@ -3,14 +3,13 @@
 module.exports = Object.freeze({
   UI_CACHE_FOLDER: 'ui',
   UI_DESC_FILENAME: 'ui.yml',
-  UI_SRC_GLOB: '**/*[!~]',
+  UI_SRC_GLOB: '**/!(*~)',
   UI_SRC_OPTS: {
+    braceExpansion: false,
     dot: true,
-    follow: true,
-    ignore: ['.git/**'],
-    nomount: true,
-    nosort: true,
-    nounique: true,
-    strict: false,
+    ignore: ['.git'],
+    objectMode: true,
+    onlyFiles: false,
+    unique: false,
   },
 })

package/lib/file.js

@@ -1,10 +1,12 @@
 'use strict'
 
-const { Readable } = require('stream')
-const { Stats } = require('fs')
+const { constants: fsc } = require('node:fs')
+const { posix: path } = require('node:path')
+const { Readable } = require('node:stream')
 const Vinyl = require('vinyl')
 
 const DEFAULT_FILE_MODE = 0o100666 & ~process.umask()
+const invariably = { true: () => true, false: () => false }
 
 class File extends Vinyl {
   get path () {
@@ -27,21 +29,92 @@ class File extends Vinyl {
 class MemoryFile extends File {
   constructor (file) {
     const contents = file.contents || Buffer.alloc(0)
-    const stat = Object.assign(new Stats(), { mode: DEFAULT_FILE_MODE, mtime: undefined, size: contents.length })
+    const stat = {
+      mode: DEFAULT_FILE_MODE,
+      size: contents.length,
+      isDirectory: invariably.false,
+      isFile: invariably.true,
+      isSymbolicLink: invariably.false,
+    }
     super(Object.assign({}, file, { contents, stat }))
   }
 }
 
-class ReadableFile extends Readable {
-  constructor (file) {
-    super({ objectMode: true })
-    this._file = file
+class ZipReadable extends Readable {
+  constructor (zipFile, options = {}) {
+    super({ objectMode: true, highWaterMark: 1 })
+    if ((this._closeable = (this._zipFile = zipFile).reader.fd != null) && !zipFile.autoClose) {
+      throw new Error('ZipReadable requires file-based ZipFile to be initialized with autoClose:true option')
+    }
+    if (!zipFile.lazyEntries) {
+      throw new Error('ZipReadable requires ZipFile to be initialized with lazyEntries:true option')
+    }
+    if ((this._startPath = options.startPath) && (this._startPath = path.join('/', this._startPath + '/')) !== '/') {
+      this._startPath = this._startPath.slice(1)
+    } else {
+      this._startPath = undefined
+    }
+    this._init()
+  }
+
+  _init () {
+    const zipFile = this._zipFile
+    zipFile
+      .on('entry', (entry) => {
+        const mode = this.getFileMode(entry)
+        if ((mode & fsc.S_IFMT) === fsc.S_IFDIR) return zipFile.readEntry()
+        let path_ = entry.fileName
+        if (this._startPath) {
+          if (path_.length < this._startPath.length || !path_.startsWith(this._startPath)) return zipFile.readEntry()
+          path_ = path_.slice(this._startPath.length)
+        }
+        const isLink = (mode & fsc.S_IFMT) === fsc.S_IFLNK
+        const stat = {
+          mode,
+          mtime: entry.getLastModDate(),
+          size: entry.uncompressedSize,
+          isDirectory: invariably.false,
+          isFile: invariably[!isLink],
+          isSymbolicLink: invariably[isLink],
+        }
+        const file = { path: path_, stat }
+        if (stat.size === 0) {
+          file.contents = Buffer.alloc(0)
+          this.push(new File(file))
+        } else {
+          zipFile.openReadStream(entry, (readErr, readStream) => {
+            if (readErr) {
+              zipFile.close()
+              this.emit('error', readErr)
+              return
+            }
+            if (isLink) {
+              const buffer = []
+              readStream
+                .on('data', (chunk) => buffer.push(chunk))
+                .on('error', (readStreamErr) => this.emit('error', readStreamErr))
+                .on('end', () => {
+                  file.symlink = (buffer.length === 1 ? buffer[0] : Buffer.concat(buffer)).toString()
+                  this.push(new File(file))
+                })
+            } else {
+              file.contents = readStream
+              this.push(new File(file))
+            }
+          })
+        }
+      })
+      .on(this._closeable ? 'close' : 'end', () => zipFile.emittedError || this.push(null))
+  }
+
+  _read (_n) {
+    this._zipFile.readEntry()
   }
 
-  _read () {
-    this.push(this._file)
-    this.push((this._file = null))
+  getFileMode ({ externalFileAttributes }) {
+    const attr = externalFileAttributes >> 16 || 33188
+    return [448, 56, 7].map((mask) => attr & mask).reduce((a, b) => a + b, attr & fsc.S_IFMT)
   }
 }
 
-module.exports = { File, MemoryFile, ReadableFile }
+module.exports = { File, MemoryFile, ZipReadable }

package/lib/load-ui.js

@@ -1,23 +1,26 @@
 'use strict'
 
 const { compile: bracesToGroup } = require('braces')
-const { createHash } = require('crypto')
+const { createHash } = require('node:crypto')
 const expandPath = require('@antora/expand-path-helper')
-const { File, MemoryFile, ReadableFile } = require('./file')
-const { promises: fsp } = require('fs')
+const { File, MemoryFile, ZipReadable } = require('./file')
+const { promises: fsp } = require('node:fs')
 const { concat: get } = require('simple-get')
 const getCacheDir = require('cache-directory')
-const globStream = require('glob-stream')
-const ospath = require('path')
+const { globStream } = require('fast-glob')
+const { inspect } = require('node:util')
+const invariably = { false: () => false, void: () => undefined }
+const ospath = require('node:path')
 const { posix: path } = ospath
 const picomatch = require('picomatch')
 const posixify = ospath.sep === '\\' ? (p) => p.replace(/\\/g, '/') : undefined
-const { pipeline, Transform, Writable } = require('stream')
+const { pipeline, PassThrough, Writable } = require('node:stream')
+const drain = () => new Writable({ objectMode: true, write: (_chunk, _enc, next) => next() })
 const forEach = (write, final) => new Writable({ objectMode: true, write, final })
-const map = (transform) => new Transform({ objectMode: true, transform })
+const through = () => new PassThrough({ objectMode: true })
 const UiCatalog = require('./ui-catalog')
 const yaml = require('js-yaml')
-const vzip = require('gulp-vinyl-zip')
+const yauzl = require('yauzl')
 
 const STATIC_FILE_MATCHER_OPTS = {
   expandRange: (begin, end, step, opts) => bracesToGroup(opts ? `{${begin}..${end}..${step}}` : `{${begin}..${end}}`),
@@ -82,9 +85,9 @@ async function loadUi (playbook) {
       return fetch && bundle.snapshot
         ? downloadBundle(bundleUrl, cachePath, createAgent(bundleUrl, playbook.network || {}))
         : fsp.stat(cachePath).then(
-          (stat) => new File({ path: cachePath, stat }),
-          () => downloadBundle(bundleUrl, cachePath, createAgent(bundleUrl, playbook.network || {}))
-        )
+            (stat) => new File({ path: cachePath, stat }),
+            () => downloadBundle(bundleUrl, cachePath, createAgent(bundleUrl, playbook.network || {}))
+          )
     })
   } else {
     const localPath = expandPath(bundleUrl, { dot: startDir })
@@ -103,20 +106,15 @@ async function loadUi (playbook) {
       new Promise((resolve, reject) =>
         bundleFile.isDirectory()
           ? srcFs(ospath.join(bundleFile.path, bundle.startPath || '', '.')).then(resolve, reject)
-          : vzip
-            .src(bundleFile.path)
-            .on('error', (err) => reject(Object.assign(err, { message: `not a valid zip file; ${err.message}` })))
-            .pipe(selectFilesStartingFrom(bundle.startPath))
-            .pipe(bufferizeContents())
-            .on('error', reject)
-            .pipe(collectFiles(resolve))
+          : srcZip(bundleFile.path, { startPath: bundle.startPath })
+              .on('error', (err) => reject(Object.assign(err, { message: `not a valid zip file; ${err.message}` })))
+              .pipe(bufferizeContentsAndCollectFiles(resolve))
+              .on('error', reject)
       ).catch((err) => {
-        const errWrapper = new Error(
+        const msg =
           `Failed to read UI ${bundleFile.isDirectory() ? 'directory' : 'bundle'}: ` +
-            (bundleUrl === bundleFile.path ? bundleUrl : `${bundleFile.path} (resolved from url: ${bundleUrl})`)
-        )
-        errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
-        throw errWrapper
+          (bundleUrl === bundleFile.path ? bundleUrl : `${bundleFile.path} (resolved from url: ${bundleUrl})`)
+        throw transformError(err, msg)
       })
     ),
     srcSupplementalFiles(supplementalFilesSpec, startDir),
@@ -175,13 +173,12 @@ function createAgent (url, { httpProxy, httpsProxy, noProxy }) {
 function downloadBundle (url, to, agent) {
   return new Promise((resolve, reject) => {
     get({ url, agent }, (err, response, contents) => {
-      if (err) reject(err)
+      if (err) return reject(err)
       if (response.statusCode !== 200) {
         const message = `Response code ${response.statusCode} (${response.statusMessage})`
         return reject(Object.assign(new Error(message), { name: 'HTTPError' }))
       }
-      new ReadableFile(new MemoryFile({ path: ospath.basename(to), contents }))
-        .pipe(vzip.src())
+      srcZip(contents, { testOnly: true })
         .on('error', (err) =>
           reject(Object.assign(err, { message: `not a valid zip file; ${err.message}`, summary: 'Invalid UI bundle' }))
         )
@@ -189,110 +186,52 @@ function downloadBundle (url, to, agent) {
           fsp
             .mkdir(ospath.dirname(to), { recursive: true })
             .then(() => fsp.writeFile(to, contents))
-            .then(() => resolve(new File({ path: to, stat: { isDirectory: () => false } })))
+            .then(() => resolve(new File({ path: to, stat: { isDirectory: invariably.false } })))
         )
     })
   }).catch((err) => {
-    const errWrapper = new Error(`${err.summary || 'Failed to download UI bundle'}: ${url}`)
-    errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
-    throw errWrapper
-  })
-}
-
-function selectFilesStartingFrom (startPath) {
-  if (!startPath || (startPath = path.join('/', startPath + '/')) === '/') {
-    return map((file, _, next) => {
-      if (file.isNull()) {
-        next()
-      } else {
-        next(
-          null,
-          new File({ path: posixify ? posixify(file.path) : file.path, contents: file.contents, stat: file.stat })
-        )
-      }
-    })
-  } else {
-    startPath = startPath.substr(1)
-    const startPathOffset = startPath.length
-    return map((file, _, next) => {
-      if (file.isNull()) {
-        next()
-      } else {
-        const path_ = posixify ? posixify(file.path) : file.path
-        if (path_.length > startPathOffset && path_.startsWith(startPath)) {
-          next(null, new File({ path: path_.substr(startPathOffset), contents: file.contents, stat: file.stat }))
-        } else {
-          next()
-        }
-      }
-    })
-  }
-}
-
-function bufferizeContents () {
-  return map((file, _, next) => {
-    // NOTE gulp-vinyl-zip automatically converts the contents of an empty file to a Buffer
-    if (file.isStream()) {
-      const buffer = []
-      pipeline(
-        file.contents,
-        forEach((chunk, _, done) => buffer.push(chunk) && done()),
-        (err) => (err ? next(err) : next(null, Object.assign(file, { contents: Buffer.concat(buffer) })))
-      )
-    } else {
-      next(null, file)
+    const errWrapper = transformError(err, `${err.summary || 'Failed to download UI bundle'}: ${url}`)
+    if (err.code === 'ECONNRESET' || (err.message || '').toLowerCase() === 'request timed out') {
+      Object.defineProperty(errWrapper, 'recoverable', { value: true })
     }
+    throw errWrapper
   })
 }
 
-function collectFiles (resolve, files = new Map()) {
-  return forEach(
-    (file, _, done) => {
-      files.set(file.path, file)
-      done()
-    },
-    (done) => done() || resolve(files)
-  )
-}
-
 function srcSupplementalFiles (filesSpec, startDir) {
   if (!filesSpec) return new Map()
   let cwd
   return (
     Array.isArray(filesSpec)
       ? Promise.all(
-        filesSpec.reduce((accum, { path: path_, contents: contents_ }) => {
-          if (!path_) {
-            return accum
-          } else if (contents_) {
-            if (~contents_.indexOf('\n') || !EXT_RX.test(contents_)) {
-              accum.push(new MemoryFile({ path: path_, contents: Buffer.from(contents_) }))
+          filesSpec.reduce((accum, { path: path_, contents: contents_ }) => {
+            if (!path_) return accum
+            if (contents_) {
+              if (~contents_.indexOf('\n') || !EXT_RX.test(contents_)) {
+                accum.push(new MemoryFile({ path: path_, contents: Buffer.from(contents_) }))
+              } else {
+                contents_ = expandPath(contents_, { dot: startDir })
+                accum.push(
+                  fsp
+                    .stat(contents_)
+                    .then((stat) =>
+                      fsp.readFile(contents_).then((contents) => new File({ path: path_, contents, stat }))
+                    )
+                )
+              }
             } else {
-              contents_ = expandPath(contents_, { dot: startDir })
-              accum.push(
-                fsp
-                  .stat(contents_)
-                  .then((stat) =>
-                    fsp.readFile(contents_).then((contents) => new File({ path: path_, contents, stat }))
-                  )
-              )
+              accum.push(new MemoryFile({ path: path_ }))
             }
-          } else {
-            accum.push(new MemoryFile({ path: path_ }))
-          }
-          return accum
-        }, [])
-      ).then((files) => files.reduce((accum, file) => accum.set(file.path, file) && accum, new Map()))
+            return accum
+          }, [])
+        ).then((files) => files.reduce((accum, file) => accum.set(file.path, file) && accum, new Map()))
       : fsp.access((cwd = expandPath(filesSpec, { dot: startDir }))).then(() => srcFs(cwd))
   ).catch((err) => {
     const dir = cwd ? filesSpec + (filesSpec === cwd ? '' : ` (resolved to ${cwd})`) : undefined
     if (err.code === 'ENOENT' && err.path === cwd) {
       throw new Error(`Specified ui.supplemental_files directory does not exist: ${dir}`)
-    } else {
-      const errWrapper = new Error(`Failed to read ui.supplemental_files ${cwd ? `directory: ${dir}` : 'entry'}`)
-      errWrapper.stack += `\nCaused by: ${err.stack || 'unknown'}`
-      throw errWrapper
     }
+    throw transformError(err, `Failed to read ui.supplemental_files ${cwd ? `directory: ${dir}` : 'entry'}`)
   })
 }
 
@@ -307,12 +246,11 @@ function loadConfig (files, outputDir) {
     files.delete(UI_DESC_FILENAME)
     const config = camelCaseKeys(yaml.load(configFile.contents.toString()))
     const staticFiles = config.staticFiles
-    if (staticFiles && staticFiles.length) config.isStaticFile = picomatch(staticFiles, STATIC_FILE_MATCHER_OPTS)
+    if (staticFiles?.length) config.isStaticFile = picomatch(staticFiles, STATIC_FILE_MATCHER_OPTS)
     if (outputDir !== undefined) config.outputDir = outputDir
     return config
-  } else {
-    return { outputDir }
   }
+  return { outputDir }
 }
 
 function camelCaseKeys (o) {
@@ -326,7 +264,7 @@ function camelCaseKeys (o) {
 }
 
 function classifyFile (file, config) {
-  if (config.isStaticFile && config.isStaticFile(file.path)) {
+  if (typeof config.isStaticFile === 'function' && config.isStaticFile(file.path)) {
     file.type = 'static'
     file.out = resolveOut(file, '')
   } else if (file.isDot()) {
@@ -353,17 +291,15 @@ function resolveOut (file, outputDir = '_') {
 }
 
 function srcFs (cwd) {
-  return new Promise((resolve, reject, cache = Object.create(null), files = new Map(), relpathStart = cwd.length + 1) =>
+  return new Promise((resolve, reject, files = new Map()) =>
     pipeline(
-      globStream(UI_SRC_GLOB, Object.assign({ cache, cwd }, UI_SRC_OPTS)),
-      forEach(({ path: abspathPosix }, _, done) => {
-        if ((cache[abspathPosix] || {}).constructor === Array) return done() // detects some directories
-        const abspath = posixify ? ospath.normalize(abspathPosix) : abspathPosix
-        const relpath = abspath.substr(relpathStart)
-        symlinkAwareStat(abspath).then(
+      globStream(UI_SRC_GLOB, Object.assign({ cwd }, UI_SRC_OPTS)),
+      forEach(({ path: relpath, dirent }, _, done) => {
+        if (dirent.isDirectory()) return done()
+        const relpathPosix = relpath
+        const abspath = posixify ? ospath.join(cwd, (relpath = ospath.normalize(relpath))) : cwd + '/' + relpath
+        fsp.stat(abspath).then(
           (stat) => {
-            if (stat.isDirectory()) return done() // detects directories that slipped through cache check
-            const relpathPosix = posixify ? posixify(relpath) : relpath
             fsp.readFile(abspath).then(
               (contents) => {
                 files.set(relpathPosix, new File({ cwd, path: relpathPosix, contents, stat, local: true }))
@@ -374,16 +310,18 @@ function srcFs (cwd) {
               }
             )
           },
-          (statErr) => {
-            done(
-              Object.assign(statErr, {
-                message: statErr.symlink
-                  ? (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
-                    `${relpath} -> ${statErr.symlink}`
-                  : statErr.message.replace(`'${abspath}'`, relpath),
-              })
-            )
-          }
+          (statErr) =>
+            dirent.isSymbolicLink()
+              ? fsp
+                  .readlink(abspath)
+                  .then(
+                    (symlink) =>
+                      (statErr.code === 'ELOOP' ? 'ELOOP: symbolic link cycle, ' : 'ENOENT: broken symbolic link, ') +
+                      `${relpath} -> ${symlink}`,
+                    () => statErr.message.replace(`'${abspath}'`, relpath)
+                  )
+                  .then((message) => done(Object.assign(statErr, { message })))
+              : done(Object.assign(statErr, { message: statErr.message.replace(`'${abspath}'`, relpath) }))
         )
       }),
       (err) => (err ? reject(err) : resolve(files))
@@ -391,18 +329,42 @@ function srcFs (cwd) {
   )
 }
 
-function symlinkAwareStat (path_) {
-  return fsp.lstat(path_).then((lstat) => {
-    if (!lstat.isSymbolicLink()) return lstat
-    return fsp.stat(path_).catch((statErr) =>
-      fsp
-        .readlink(path_)
-        .catch(() => undefined)
-        .then((symlink) => {
-          throw Object.assign(statErr, { symlink })
-        })
-    )
+function srcZip (file, options = {}) {
+  const result = options.testOnly
+    ? forEach((file_, _, done) => (file_.isStream() ? pipeline(file_.contents, through(), drain(), done) : done()))
+    : through()
+  yauzl[file instanceof Buffer ? 'fromBuffer' : 'open'](file, { lazyEntries: true }, (err, zipFile) => {
+    if (err) return result.emit('error', err)
+    new ZipReadable(zipFile, options).pipe(result)
   })
+  return result
+}
+
+function bufferizeContentsAndCollectFiles (resolve, files = new Map()) {
+  return forEach(
+    (file, _, done) => {
+      if (file.isStream()) {
+        const buffer = []
+        file.contents
+          .on('data', (chunk) => buffer.push(chunk))
+          .on('end', () => {
+            file.contents = buffer.length === 1 ? buffer[0] : Buffer.concat(buffer)
+            files.set(file.path, file)
+            done()
+          })
+      } else {
+        files.set(file.path, file)
+        done()
+      }
+    },
+    (done) => done() || resolve(files)
+  )
+}
+
+function transformError (err, msg) {
+  const errWrapper = new Error(msg)
+  errWrapper.stack += `\nCaused by: ${err.stack ? inspect(err).replace(/^Error \[(.+)\](?=: )/, '$1') : err}`
+  return errWrapper
 }
 
 module.exports = loadUi

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@antora/ui-loader",
-  "version": "3.2.0-alpha.1",
+  "version": "3.2.0-alpha.10",
   "description": "Downloads a UI bundle, if necessary, and loads the files into a UI catalog for use in an Antora documentation pipeline.",
   "license": "MPL-2.0",
   "author": "OpenDevise Inc. (https://opendevise.com)",
@@ -11,7 +11,11 @@
     "Guillaume Grossetie <g.grossetie@gmail.com>"
   ],
   "homepage": "https://antora.org",
-  "repository": "gitlab:antora/antora",
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/antora/antora.git",
+    "directory": "packages/ui-loader"
+  },
   "bugs": {
     "url": "https://gitlab.com/antora/antora/issues"
   },
@@ -25,20 +29,20 @@
     "#constants": "./lib/constants.js"
   },
   "dependencies": {
-    "@antora/expand-path-helper": "~2.0",
+    "@antora/expand-path-helper": "~3.0",
     "braces": "~3.0",
     "cache-directory": "~2.0",
-    "glob-stream": "~7.0",
-    "gulp-vinyl-zip": "~2.5",
-    "hpagent": "~1.1",
+    "fast-glob": "~3.3",
+    "hpagent": "~1.2",
     "js-yaml": "~4.1",
-    "picomatch": "~2.3",
+    "picomatch": "~4.0",
     "should-proxy": "~1.0",
     "simple-get": "~4.0",
-    "vinyl": "~2.2"
+    "vinyl": "~3.0",
+    "yauzl": "~3.1"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.0.0"
   },
   "files": [
     "lib/"
@@ -54,7 +58,7 @@
   ],
   "scripts": {
     "test": "_mocha",
-    "prepublishOnly": "node $npm_config_local_prefix/npm/prepublishOnly.js",
-    "postpublish": "node $npm_config_local_prefix/npm/postpublish.js"
+    "prepublishOnly": "npx -y downdoc@latest --prepublish",
+    "postpublish": "npx -y downdoc@latest --postpublish"
   }
 }