@thegetty/quire-cli 1.0.0-rc.11 → 1.0.0-rc.12

package/CHANGELOG.md

@@ -18,6 +18,16 @@ Changelog entries are classified using the following labels:
 
 - `Removed`: for deprecated features removed in this release
 
+## [1.0.0-rc.12]
+
+### Added
+- Handling for single-page PDFs
+- Paged and Prince plugins to map quire pages to PDF page sections 
+
+### Changed
+
+- `quire clean` also removes `.11ty-vite` in the project root
+
 ## [1.0.0-rc.11]
 
 ### Bumped

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@thegetty/quire-cli",
   "description": "Quire command-line interface",
-  "version": "1.0.0-rc.11",
+  "version": "1.0.0-rc.12",
   "author": "Getty Digital",
   "license": "SEE LICENSE IN https://github.com/thegetty/quire/blob/main/LICENSE",
   "bugs": {
@@ -45,6 +45,7 @@
     "#lib/*.js": "./src/lib/*.js"
   },
   "dependencies": {
+    "ajv": "^8.16.0",
     "boxen": "^7.0.1",
     "chalk": "^5.2.0",
     "commander": "^10.0.0",
@@ -58,6 +59,7 @@
     "i18next": "^22.4.9",
     "inquirer": "^9.1.4",
     "install-npm-version": "^1.0.8",
+    "js-yaml": "^4.1.0",
     "loglevel": "^1.8.1",
     "open": "^8.4.0",
     "ora": "^6.1.2",

package/src/commands/pdf.js

@@ -1,10 +1,51 @@
-import Command from '#src/Command.js'
 import { paths, projectRoot  } from '#lib/11ty/index.js'
+import Command from '#src/Command.js'
 import fs from 'fs-extra'
 import libPdf from '#lib/pdf/index.js'
 import open from 'open'
 import path from 'node:path'
-// import testcwd from '#helpers/test-cwd.js'
+import { validateUserConfig } from '../../../11ty/_plugins/globalData/validator.js'
+import yaml from 'js-yaml'
+
+/**
+ * @function loadConfig(path) - loads and validates quire config, returns an empty object if not found
+ *  
+ * @param {path} string - file path to config file
+ * 
+ * @todo refactor loading configs to a separate module,
+ * which uses the same validator(s) as the build proceess
+ * 
+ * @return {Object|undefined} User configuration object or undefined
+ */
+function loadConfig(confPath) {
+  if (!fs.existsSync(confPath)) {
+    return undefined
+  }
+
+  const data = fs.readFileSync(confPath)
+  let config = yaml.load(data)
+
+  // NB: Schemas may be project specific so the CLI must loads from project root rather than package root
+  const schemaPath = path.join(projectRoot,'_plugins','schemas','config.json')
+
+  // TODO: Check the project version string for whether we should check the schema
+  if (fs.existsSync(schemaPath)) {
+
+    const sch = fs.readFileSync(schemaPath)
+    const schema = JSON.parse(sch)      
+
+    try {
+      config = validateUserConfig('config',config,{config: schema})    
+    } catch (err) {
+      console.error(err)
+      process.exit(1)
+    }
+  }
+
+  return config
+}
+
+const quireConfig = loadConfig(path.join(projectRoot,'content','_data','config.yaml'))
 
 /**
  * Quire CLI `pdf` Command
@@ -33,6 +74,7 @@ export default class PDFCommand extends Command {
   }
 
   constructor() {
+    const config = loadConfig(path.join(projectRoot,'content','_data','config.yaml'))
     super(PDFCommand.definition)
   }
 
@@ -41,22 +83,28 @@ export default class PDFCommand extends Command {
       console.debug('[CLI] Command \'%s\' called with options %o', this.name(), options)
     }
 
-    const input = path.join(projectRoot, paths.output, 'pdf.html')
+    const publicationInput = path.join(projectRoot, paths.output, 'pdf.html')
+    const coversInput = path.join(projectRoot, paths.output, 'pdf-covers.html')
 
-    if (!fs.existsSync(input)) {
-      console.error(`Unable to find PDF input at ${input}\nPlease first run the 'quire build' command.`)
-      return
+    if (quireConfig===undefined) {
+      console.error(`[quire pdf]: ERROR Unable to find a configuration file at ${path.join(projectRoot,'content','_data','config.yaml')}\nIs the command being run in a quire project?`)
+      process.exit(1)
+    }
+
+    if (!fs.existsSync(publicationInput)) {
+      console.error(`[quire pdf]: ERROR Unable to find PDF input at ${publicationInput}\nPlease first run the 'quire build' command.`)
+      process.exit(1)
     }
 
-    const output = path.join(projectRoot, `${options.lib}.pdf`)
+    const output = quireConfig.pdf !== undefined ? path.join(paths.output, quireConfig.pdf.outputDir, `${quireConfig.pdf.filename}.pdf`) : path.join(projectRoot, `${options.lib}.pdf`)
 
-    const pdfLib = await libPdf(options.lib, { debug: options.debug })
-    await pdfLib(input, output)
+    const pdfLib = await libPdf(options.lib, { ...options, pdfConfig: quireConfig.pdf })
+    await pdfLib(publicationInput, coversInput, output)
 
     try {
       if (fs.existsSync(output) && options.open) open(output)
     } catch (error) {
-      console.error(error)
+      console.error(`[quire pdf]: ERROR`,error)
       process.exit(1)
     }
   }

package/src/helpers/clean.js

@@ -9,6 +9,7 @@ import path from 'node:path'
  */
 export async function clean (projectRoot, paths, options = {}) {
   const pathsToClean = [
+    path.join(projectRoot, '.11ty-vite'),
     path.join(projectRoot, paths.epub),
     path.join(projectRoot, paths.output),
     path.join(projectRoot, '*.epub'),

package/src/lib/pdf/README.md

@@ -1,12 +1,19 @@
 ## CLI lib/PDF Module
 
-This module implements a façade for PDF generation libries.
-The module exports a single method that accepts a `lib` option and delegates to a façade for the specified PDF library.
+This module provides an abstraction to ease PDF generation across PrinceXML and Paged.js. The exported module dynamically loads a wrapper to align the libraries' JS APIs by exporting a single method that accepts a `lib` option and returns an async function that takes input, output, and option params.
+
+The module also provides plugins for Prince and Paged.js to map quire webpages to PDF pages. In both cases this is achieved after PDF rendering by querying the HTML document that was printed for `.quire-page` elements and using the PDF generator's APIs to determine content page ids, page data like titles and contributors, and first / last pages. They then use a simple stripping algorithm with `pdf-lib` to split the pages for `--page-pdf` flagged runs.
 
 ### Paged.js Façade
 
-See the [`Paged.js` documentation](https://gitlab.coko.foundation/pagedjs/).
+We use [`pagedjs-cli`](https://gitlab.coko.foundation/pagedjs/pagedjs-cli), which adds a headless to paged.js to facilitate serializing to PDF files.
+
+For more details on the PDF generating API see the [`Paged.js` documentation](https://gitlab.coko.foundation/pagedjs/). See paged.js's [hooks documentation](https://pagedjs.org/documentation/10-handlers-hooks-and-custom-javascript/) for details on the `afterRendered` hook that quire uses to generate the PDF page map.
 
 ### Prince XML Façade
 
-See the [Prince Command-line Reference](https://www.princexml.com/doc/command-line/).
+The Prince abstraction wraps the command line execution of the Prince executable. 
+
+See the [Prince Command-line Reference](https://www.princexml.com/doc/command-line/). Prince's [scripting documentation](https://www.princexml.com/doc/javascript/) has details on its runtime Javascript implementation, *which is only compatible up to ES5*.
+
+The Prince plugin passes page map data as JSON to STDOUT.

package/src/lib/pdf/index.js

@@ -17,14 +17,14 @@ export default async (name = 'pagedjs', options = {}) => {
     case 'paged':
     case 'pagedjs': {
       lib.name = 'Paged.js'
-      lib.options = { debug: options.debug }
+      lib.options = options
       lib.path = path.join(__dirname, 'paged.js')
       break
     }
     case 'prince':
     case 'princexml': {
       lib.name = 'Prince'
-      lib.options = { debug: options.debug, verbose: options.verbose }
+      lib.options = options
       lib.path = path.join(__dirname, 'prince.js')
       break
     }
@@ -35,8 +35,8 @@ export default async (name = 'pagedjs', options = {}) => {
 
   const { default: pdfLib } = await dynamicImport(lib.path)
 
-  return async (input, output) => {
+  return async (publicationInput, coversInput, output) => {
     console.info(`[CLI:lib/pdf] generating PDF using ${lib.name}`)
-    return await pdfLib(input, output, lib.options)
+    return await pdfLib(publicationInput, coversInput, output, lib.options)
   }
 }

package/src/lib/pdf/paged.js

@@ -1,19 +1,38 @@
 import Printer from 'pagedjs-cli'
+
 import fs from 'fs-extra'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { splitPdf } from './split.js'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+// FIXME: This module swallows errors currently.
 
 /**
- * A façade module for interacting with Paged.js
+ * A façade module for interacting with Paged.js and pagedjs-cli  
  * @see https://gitlab.coko.foundation/pagedjs/
  */
-export default async (input, output, options = {}) => {
+export default async (publicationInput, coversInput, output, options = {}) => {
   /**
    * Configure the Paged.js Printer options
    * @see https://gitlab.coko.foundation/pagedjs/pagedjs-cli/-/blob/main/src/cli.js
    */
+
+  let additionalScripts = []
+  
+  const { pdfConfig } = options
+
+  additionalScripts.push( path.join(__dirname, 'pagedPlugin.js') )
+
   const printerOptions = {
     allowLocal: true,
     debug: options.debug || false,
     enableWarnings: options.debug || false,
+    closeAfter: false,
+    additionalScripts,
   }
 
   if (options.debug) {
@@ -21,12 +40,12 @@ export default async (input, output, options = {}) => {
     console.debug(`[CLI:lib/pdf/pagedjs] Printer options\n${optionsOutput}`)
   }
 
-  const printer = new Printer(printerOptions)
+  let printer = new Printer(printerOptions)
 
-  printer.on('page', (page) => {
+  printer.on('page', (page,pageElement,breakToken) => {
     if (page.position === 0) {
       console.info(`[CLI:lib/pdf/pagedjs] loaded`)
-    }
+    } 
   })
 
   printer.on('rendered', (msg) => {
@@ -54,18 +73,78 @@ export default async (input, output, options = {}) => {
   }
 
   try {
-    console.info(`[CLI:lib/pdf/pagedjs] printing ${input}`)
+    console.info(`[CLI:lib/pdf/pagedjs] printing ${publicationInput}`)
 
-    const file = await printer.pdf(input, pdfOptions)
+    const file = await printer.pdf(publicationInput, pdfOptions)
       .catch((error) => console.error(error))
 
-    printer.close()
+    let pageMap
+
+    // Now it's printed, create the pageMap by running JS in the printer's context
+    let coversFile
+
+    console.info(`[CLI:lib/pdf/pagedjs] generating page map`)
+    const pages = await printer.browser.pages()
+
+    if (pages.length > 0) {
+      pageMap = await pages[pages.length - 1].evaluate(() => {
+        // Retrieves the pageMap from our plugin
+        return window.pageMap ?? {} // eslint-disable-line no-undef
+      })
+    }
+
+    if ( pdfConfig?.pagePDF?.coverPage===true && fs.existsSync(coversInput) ) {
+      console.info(`[CLI:lib/pdf/pagedjs] printing ${coversInput}`)
+
+      const coverPrinter = new Printer(printerOptions)
+
+      coversFile = await coverPrinter.pdf(coversInput, pdfOptions)
+        .catch((error) => console.error(error))
+
+      const coverPages = await coverPrinter.browser.pages()
+
+      if (coverPages.length > 0) {
+        const coversMap = await coverPages[coverPages.length - 1].evaluate(() => {
+          // Retrieves the pageMap from our plugin
+          return window.pageMap ?? {} // eslint-disable-line no-undef
+        })
+
+        Object.values(coversMap).forEach( cov => {
+          if (cov.id in pageMap) {
+            pageMap[cov.id].coverPage = cov.startPage           
+          }
+        })
+
+      }
+
+      coverPrinter.close()
+    }      
+
+    // Leave the printer open for debug logs
+    if (!options.debug) {
+      printer.close()    
+    }
 
     if (file && output) {
+      console.info(`[CLI:lib/pdf/pagedjs] writing file(s)`)
+
+      const { dir } = path.parse(output)
+      if (!fs.existsSync(dir)) { 
+        fs.mkdirsSync(dir)
+      }
+
       await fs.promises.writeFile(output, file)
         .catch((error) => console.error(error))
+
+      const files = await splitPdf(file,coversFile,pageMap,options.pdfConfig)
+
+      Object.entries(files).forEach( async ([filePath,pagePdf]) => {
+        await fs.promises.writeFile(filePath,pagePdf)
+          .catch((error) => console.error(error))
+      })
     }
+
   } catch (ERR_FILE_NOT_FOUND) {
-    console.error(`[CLI:lib/pdf/pagedjs] file not found ${input}`)
+    console.error(`[CLI:lib/pdf/pagedjs] file not found ${publicationInput}`)
   }
 }

package/src/lib/pdf/pagedPlugin.js

@@ -0,0 +1,49 @@
+/**
+ * @class pageTableMapper
+ * 
+ * Responds to [paged.js afterRender hook](https://pagedjs.org/documentation/10-handlers-hooks-and-custom-javascript/) to map the PDF's page table and store it in window.pageMap
+ * 
+ */
+class pageTableMapper extends Paged.Handler { // eslint-disable-line no-undef
+  constructor(chunker, polisher, caller) {
+    super(chunker, polisher, caller)
+  }
+
+  /**
+   * @function afterRendered - fires after all pages are laid out and PDF data is available
+   * 
+   * @param {Array<Page>} pages - Pages rendered from the document
+   */
+  afterRendered(pages) {
+    let pageMap = {}
+    let webpageKey
+
+    // Iterate pages, build lookup by finding the website page on each PDF page
+    for (const p of pages) {
+      const quirePageElement = p.element.querySelector('.quire-page')
+
+      if (!quirePageElement) { continue }
+      if (quirePageElement.dataset.pagePdf !== 'true') { continue }
+
+      const quirePageId = quirePageElement.dataset.id ?? quirePageElement.id
+
+      if (webpageKey !== quirePageId) {
+        webpageKey = quirePageId
+
+        const title = quirePageElement.dataset.pdfCoverPageTitle
+
+        let data = { id: webpageKey, startPage: p.position, endPage: p.position, title, }
+
+        pageMap[webpageKey] = data
+
+      } else {
+        pageMap[webpageKey].endPage = p.position
+      }
+    }
+
+    window.pageMap = pageMap // eslint-disable-line no-undef
+
+  }
+}
+
+Paged.registerHandlers(pageTableMapper) // eslint-disable-line no-undef

package/src/lib/pdf/prince.js

@@ -1,23 +1,97 @@
 import { execa } from 'execa'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { splitPdf } from './split.js'
+import fs from 'fs-extra'
+
 import which from '#helpers/which.js'
 
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
 /**
  * A façade module for interacting with the Prince CLI.
  * @see https://www.princexml.com/doc/command-line/
  */
-export default async (input, output, options = {}) => {
+export default async (publicationInput, coversInput, output, options = {}) => {
   which('prince')
 
   /**
    * @see https://www.princexml.com/doc/command-line/#options
    */
+  
+  const { pdfConfig } = options
+
+  // These options run once to get the map pages to essays
+  const pageMapOptions = [
+    `--script=${ path.join(__dirname, 'princePlugin.js') }`,
+    `--output=${output}`,
+  ]
+
+  // These options are for the actual user-facing PDF
   const cmdOptions = [
     `--output=${output}`,
   ]
 
-  if (options.debug) cmdOptions.push('--debug')
-  if (options.verbose) cmdOptions.push('--verbose')
+  if (options.debug) {
+    pageMapOptions.push('--debug')
+    cmdOptions.push('--debug')
+  }
+  
+  if (options.verbose) {
+    pageMapOptions.push('--verbose')
+    cmdOptions.push('--verbose')
+  }
+
+  const { dir } = path.parse(output)
+  if (!fs.existsSync(dir)) { 
+    fs.mkdirsSync(dir)
+  }
+  
+  // Execute the page mapping PDF build  
+  let pageMap = {}
+  try {
+    const pageMapOutput = await execa('prince', [...pageMapOptions, publicationInput])  
+    pageMap = JSON.parse(pageMapOutput.stdout)
+  } catch (err) {
+    console.error(`Generating the PDF page map failed with the error ${err.stderr}`)
+    process.exit(1)
+  }
+
+  let coversData
+
+  if (pdfConfig?.pagePDF?.coverPage === true) {
+
+    const coversPageMapOutput = await execa('prince', [...pageMapOptions, coversInput])
+    const coversMap = JSON.parse(coversPageMapOutput.stdout)
+
+    for (const pageId of Object.keys(coversMap))  {
+      if (pageId in pageMap) {
+        pageMap[pageId].coverPage = coversMap[pageId].startPage       
+      }
+    }
+
+    coversData = fs.readFileSync(output,null)
+
+  }
+
+  let stderror,stdout
+
+  try {
+    ({ stderror, stdout } = await execa('prince', [...cmdOptions, publicationInput]))
+  } catch (err) {
+    console.error(`Printing the PDF failed with the error ${err.stderr}`)
+    process.exit(1)
+  }
+
+  const pdfData = fs.readFileSync(output,null)
+  
+  let files = await splitPdf(pdfData,coversData,pageMap,pdfConfig)
+  Object.entries(files).forEach( async ([filePath,pagePdf]) => {
+    await fs.promises.writeFile(filePath,pagePdf)
+      .catch((error) => console.error(error))
+  })
 
-  const { stderror, stdout } = await execa('prince', [...cmdOptions, input])
   return { stderror, stdout }
 }

package/src/lib/pdf/princePlugin.js

@@ -0,0 +1,35 @@
+/**
+ * @function generatePageMap()
+ * 
+ * Walks the PDF-HTML map after printing and serializes the mapped output to STDOUT
+ */
+function generatePageMap() {
+  
+  const els = document.querySelectorAll('.quire-page[data-page-pdf=true]') // eslint-disable-line no-undef
+  let pageMap = {}
+
+  for (let i=0;i < els.length;i++) {
+  
+    const el = els[i]
+    const boxes = el.getPrinceBoxes()
+
+    if (boxes.length < 1) { continue }
+
+    let data = { id: el.getAttribute('data-id') || el.id, title: el.getAttribute('data-footer-page-title'), startPage: boxes[0].pageNum - 1, endPage: boxes[boxes.length-1].pageNum - 1 }
+
+    let pageKey = el.getAttribute('data-id') || el.id
+
+    pageMap[pageKey] = data 
+
+    if ( el.getAttribute('data-pdf-cover-page') !== 'true' ) {
+      continue
+    }
+
+  }
+
+  console.log(JSON.stringify(pageMap))
+
+}
+
+Prince.trackBoxes = true // eslint-disable-line no-undef
+Prince.oncomplete = generatePageMap // eslint-disable-line no-undef

package/src/lib/pdf/split.js

@@ -0,0 +1,61 @@
+import path from 'node:path'
+
+import { PDFDocument } from 'pdf-lib'
+import { paths } from '#lib/11ty/index.js'
+  
+/**
+ * @function splitPdf(file,pageMap) -- sections out individual PDFs from `file` according to `pageMap`
+ * 
+ * @param {ArrayBuffer} file - PDF file to split
+ * @param {Object} pageMap - page map to split PDf by
+ * 
+ * Creates individual PDFs from by copying `file` (so boxes are already set) and stripping pages out of the range (in reverse order to retain the index sequence)
+ * Returns a map of file paths to PDF binary data for serialization
+ */
+
+export async function splitPdf(file,coversFile,pageMap,pdfConfig) {
+
+  if (!pdfConfig) { 
+    return {}
+  }
+
+  const { filename, outputDir } = pdfConfig
+
+  const pdfDoc = await PDFDocument.load(file)
+
+  const coversDoc = (coversFile !== undefined) ? await PDFDocument.load(coversFile) : undefined
+
+  let resultFiles = {}
+
+  for ( const [pageId, pageConfig] of Object.entries(pageMap) ) {
+    const { endPage, startPage, coverPage } = pageConfig
+
+    // TODO: Set the PDF's sectional doc metadata
+    
+    const sectionDoc = await pdfDoc.copy()
+
+    for (let p=pdfDoc.getPageCount() - 1; p > endPage; --p) {
+      sectionDoc.removePage(p)
+    }
+    for (let q=startPage - 1; q >= 0; --q) {
+      sectionDoc.removePage(q)
+    }
+
+    if (coversDoc && coverPage !== undefined && coverPage >= 0) {
+      // NB: `copyPages()` sets page sizing + other metadata on the way into the target PDF
+      const cover = await sectionDoc.copyPages(coversDoc,[coverPage])
+      sectionDoc.insertPage(0,cover[0])
+    }
+
+    const section = await sectionDoc.save()
+
+    const sectionId = pageId.replace(/^page-/g,'')
+    const sectionFn = `${filename}-${sectionId}.pdf`
+    const sectionFp = path.join( paths.output, outputDir, sectionFn )
+
+    resultFiles[sectionFp] = section
+  }
+
+  return resultFiles
+}
+