@abreen/tada 1.0.0

package/webpack/pagefind-plugin.js

@@ -0,0 +1,379 @@
+const fs = require('fs');
+const path = require('path');
+const { makeLogger } = require('./log');
+const { collectReachableSiteAssets } = require('./reachability');
+const ContentWatchPlugin = require('./content-watch-plugin');
+const {
+  getBuildContentFiles,
+  getContentDir,
+  normalizeOutputPath,
+} = require('./util');
+const { assertMutoolAvailable, extractPdfPages } = require('./pdf-text');
+
+const log = makeLogger(__filename);
+const PAGEFIND_VERBOSE = process.env.TADA_LOG_LEVEL === 'debug';
+const PAGEFIND_OUTPUT_SUBDIR = 'pagefind';
+
+let pagefindModulePromise = null;
+
+function getPagefind() {
+  if (!pagefindModulePromise) {
+    pagefindModulePromise = import('pagefind');
+  }
+  return pagefindModulePromise;
+}
+
+function formatPagefindErrors(step, errors) {
+  if (!errors?.length) {
+    return null;
+  }
+  return `${step} failed: ${errors.join(' | ')}`;
+}
+
+function readAssetContent(outputFileSystem, distPath, sourcePath) {
+  const filePath = path.join(distPath, sourcePath);
+  return String(outputFileSystem.readFileSync(filePath, 'utf-8'));
+}
+
+async function addHtmlFile(index, htmlFile) {
+  const { errors: addErrors } = await index.addHTMLFile(htmlFile);
+  const addError = formatPagefindErrors(
+    `index.addHTMLFile(${htmlFile.sourcePath})`,
+    addErrors,
+  );
+  if (addError) {
+    throw new Error(addError);
+  }
+}
+
+async function addPdfRecord(index, record, sourcePath) {
+  const { errors: addErrors } = await index.addCustomRecord(record);
+  const addError = formatPagefindErrors(
+    `index.addCustomRecord(${sourcePath})`,
+    addErrors,
+  );
+  if (addError) {
+    throw new Error(addError);
+  }
+}
+
+function getPdfSourceByOutputPath(siteVariables) {
+  const contentDir = getContentDir();
+  const contentFiles = getBuildContentFiles(
+    contentDir,
+    Object.keys(siteVariables?.codeLanguages || {}),
+  );
+  const pdfFiles = contentFiles.filter(
+    filePath => path.extname(filePath).toLowerCase() === '.pdf',
+  );
+
+  return new Map(
+    pdfFiles.map(filePath => {
+      const relPath = path.relative(contentDir, filePath);
+      return [normalizeOutputPath(`/${relPath}`), filePath];
+    }),
+  );
+}
+
+function collectIndexTargets(
+  htmlAssetsByPath,
+  siteVariables,
+  pdfSourceByOutputPath,
+) {
+  if (htmlAssetsByPath.size === 0) {
+    return { reachableHtmlPaths: [], reachablePdfPaths: [] };
+  }
+
+  return collectReachableSiteAssets({
+    htmlAssetsByPath,
+    knownPdfPaths: new Set(pdfSourceByOutputPath.keys()),
+    rootPath: 'index.html',
+    basePath: siteVariables?.basePath || '/',
+  });
+}
+
+async function buildIndex({
+  distPath,
+  htmlAssetsByPath,
+  reachableHtmlPaths,
+  reachablePdfPaths,
+  pdfSourceByOutputPath,
+  loadPagefind = getPagefind,
+  checkMutool = assertMutoolAvailable,
+  extractPages = extractPdfPages,
+}) {
+  const pagefind = await loadPagefind();
+  const { index, errors: createErrors } = await pagefind.createIndex({
+    keepIndexUrl: true,
+    verbose: PAGEFIND_VERBOSE,
+  });
+  const createError = formatPagefindErrors(
+    'pagefind.createIndex()',
+    createErrors,
+  );
+  if (createError) {
+    throw new Error(createError);
+  }
+  if (!index) {
+    throw new Error('pagefind.createIndex() did not return an index');
+  }
+
+  try {
+    for (const sourcePath of reachableHtmlPaths) {
+      await addHtmlFile(index, {
+        sourcePath,
+        content: htmlAssetsByPath.get(sourcePath),
+      });
+    }
+
+    let mutoolAvailable = true;
+    if (reachablePdfPaths.length > 0) {
+      try {
+        await checkMutool();
+      } catch {
+        mutoolAvailable = false;
+        log.warn`mutool was not found; search results will not include PDFs`;
+      }
+    }
+
+    for (const pdfPath of mutoolAvailable ? reachablePdfPaths : []) {
+      const sourceFilePath = pdfSourceByOutputPath.get(pdfPath);
+      if (!sourceFilePath) {
+        continue;
+      }
+
+      const { pages, hasExtractedText } = await extractPages(sourceFilePath);
+      const title = path.posix.basename(pdfPath);
+
+      if (!hasExtractedText) {
+        await addPdfRecord(
+          index,
+          { url: pdfPath, content: title, language: 'en', meta: { title } },
+          pdfPath,
+        );
+        continue;
+      }
+
+      for (const page of pages) {
+        await addPdfRecord(
+          index,
+          {
+            url: `${pdfPath}#page=${page.pageNumber}`,
+            content: page.content,
+            language: 'en',
+            meta: { title, page: String(page.pageNumber) },
+          },
+          `${pdfPath}#page=${page.pageNumber}`,
+        );
+      }
+    }
+
+    const { errors: writeErrors } = await index.writeFiles({
+      outputPath: path.join(distPath, PAGEFIND_OUTPUT_SUBDIR),
+    });
+    const writeError = formatPagefindErrors('index.writeFiles()', writeErrors);
+    if (writeError) {
+      throw new Error(writeError);
+    }
+  } finally {
+    await index.deleteIndex().catch(() => null);
+  }
+}
+
+class PagefindPlugin {
+  constructor(siteVariables) {
+    this.siteVariables = siteVariables || {};
+    this.watchRunInProgress = false;
+    this.watchRunQueued = false;
+    this.lastDistPath = null;
+    this.htmlCacheByAssetPath = new Map();
+  }
+
+  getHtmlAssetsByPath(compilation, distPath, outputFileSystem) {
+    return new Map(
+      compilation
+        .getAssets()
+        .filter(asset => asset.name.endsWith('.html'))
+        .map(asset => [
+          asset.name.replace(/\\/g, '/'),
+          readAssetContent(
+            outputFileSystem,
+            distPath,
+            asset.name.replace(/\\/g, '/'),
+          ),
+        ]),
+    );
+  }
+
+  getIndexTargets(htmlAssetsByPath) {
+    return collectIndexTargets(
+      htmlAssetsByPath,
+      this.siteVariables,
+      getPdfSourceByOutputPath(this.siteVariables),
+    );
+  }
+
+  runWatchIndex() {
+    if (this.watchRunInProgress) {
+      this.watchRunQueued = true;
+      log.info`Indexing is still running in the background; queueing a rerun`;
+      return;
+    }
+
+    this.watchRunInProgress = true;
+    this.watchRunQueued = false;
+    const distPath = this.lastDistPath;
+    const htmlAssetsByPath = new Map(this.htmlCacheByAssetPath);
+    const pdfSourceByOutputPath = getPdfSourceByOutputPath(this.siteVariables);
+    const start = Date.now();
+
+    log.debug`Preparing search index background snapshot`;
+
+    let reachableHtmlPaths;
+    let reachablePdfPaths;
+    try {
+      ({ reachableHtmlPaths, reachablePdfPaths } = collectIndexTargets(
+        htmlAssetsByPath,
+        this.siteVariables,
+        pdfSourceByOutputPath,
+      ));
+    } catch (err) {
+      this.watchRunInProgress = false;
+      log.warn`Pagefind failed: ${err.message}`;
+      if (this.watchRunQueued) {
+        this.runWatchIndex();
+      }
+      return;
+    }
+
+    const snapshotReadyAt = Date.now();
+    log.debug`Building search index in background`;
+    buildIndex({
+      distPath,
+      htmlAssetsByPath,
+      reachableHtmlPaths,
+      reachablePdfPaths,
+      pdfSourceByOutputPath,
+    })
+      .then(() => {
+        const finishedAt = Date.now();
+        log.info`Background search index ready in ${finishedAt - snapshotReadyAt}ms (${finishedAt - start}ms total)`;
+      })
+      .catch(err => {
+        const failedAt = Date.now();
+        log.warn`Search index failed after ${failedAt - snapshotReadyAt}ms of indexing (${failedAt - start}ms total): ${err.message}`;
+      })
+      .finally(() => {
+        this.watchRunInProgress = false;
+        if (this.watchRunQueued) {
+          log.info`Starting queued Pagefind background rerun`;
+          this.runWatchIndex();
+        }
+      });
+  }
+
+  apply(compiler) {
+    compiler.hooks.afterEmit.tapAsync(
+      'PagefindPlugin',
+      (compilation, callback) => {
+        const distPath =
+          compiler.options?.output?.path ||
+          compiler.outputPath ||
+          compilation.compiler.outputPath;
+        const outputFileSystem =
+          compiler.outputFileSystem ||
+          compilation.compiler.outputFileSystem ||
+          fs;
+        const isWatch = !!compiler.watching;
+
+        if (compilation.errors.length > 0) {
+          callback();
+          return;
+        }
+
+        let htmlAssetsByPath;
+        try {
+          htmlAssetsByPath = this.getHtmlAssetsByPath(
+            compilation,
+            distPath,
+            outputFileSystem,
+          );
+        } catch (err) {
+          compilation.errors.push(err);
+          callback(err);
+          return;
+        }
+
+        if (isWatch) {
+          this.lastDistPath = distPath;
+          this.htmlCacheByAssetPath = htmlAssetsByPath;
+          callback();
+          return;
+        }
+
+        const pdfSourceByOutputPath = getPdfSourceByOutputPath(
+          this.siteVariables,
+        );
+        const start = Date.now();
+        let reachableHtmlPaths;
+        let reachablePdfPaths;
+
+        log.info`Finding reachable pages for search index`;
+        try {
+          ({ reachableHtmlPaths, reachablePdfPaths } = collectIndexTargets(
+            htmlAssetsByPath,
+            this.siteVariables,
+            pdfSourceByOutputPath,
+          ));
+        } catch (err) {
+          compilation.errors.push(err);
+          callback(err);
+          return;
+        }
+
+        const snapshotReadyAt = Date.now();
+        log.info`Building search index for ${reachableHtmlPaths.length} page(s) and ${reachablePdfPaths.length} PDF(s) after ${snapshotReadyAt - start}ms of snapshot prep`;
+        buildIndex({
+          distPath,
+          htmlAssetsByPath,
+          reachableHtmlPaths,
+          reachablePdfPaths,
+          pdfSourceByOutputPath,
+        })
+          .then(async () => {
+            try {
+              const pagefind = await getPagefind();
+              await pagefind.close();
+            } catch (_err) {
+              // Best-effort cleanup for non-watch builds.
+            }
+            const finishedAt = Date.now();
+            log.info`Search index built in ${finishedAt - snapshotReadyAt}ms (${finishedAt - start}ms total)`;
+            callback();
+          })
+          .catch(err => {
+            const failedAt = Date.now();
+            log.error`Search indexing failed after ${failedAt - snapshotReadyAt}ms of indexing (${failedAt - start}ms total): ${err.message}`;
+            compilation.errors.push(err);
+            callback(err);
+          });
+      },
+    );
+
+    compiler.hooks.done.tap('PagefindPluginWatchRun', stats => {
+      if (
+        !compiler.watching ||
+        stats.hasErrors() ||
+        ContentWatchPlugin.needsRestart()
+      ) {
+        return;
+      }
+
+      setImmediate(() => this.runWatchIndex());
+    });
+  }
+}
+
+module.exports = PagefindPlugin;
+module.exports.buildIndex = buildIndex;
+module.exports.collectIndexTargets = collectIndexTargets;

package/webpack/pagefind-plugin.test.js

@@ -0,0 +1,131 @@
+const path = require('path');
+const { describe, expect, test } = require('bun:test');
+const { buildIndex, collectIndexTargets } = require('./pagefind-plugin');
+
+function createFakePagefind(calls) {
+  const fakeIndex = {
+    addHTMLFile: async file => {
+      calls.htmlFiles?.push(file);
+      return { errors: [], file: { url: file.sourcePath, meta: {} } };
+    },
+    addCustomRecord: async record => {
+      calls.customRecords?.push(record);
+      return { errors: [], file: { url: record.url, meta: record.meta } };
+    },
+    writeFiles: async ({ outputPath }) => {
+      calls.outputPath = outputPath;
+      return { errors: [], outputPath };
+    },
+    deleteIndex: async () => {
+      calls.deleted = (calls.deleted || 0) + 1;
+    },
+  };
+
+  return async () => ({
+    createIndex: async () => ({ index: fakeIndex, errors: [] }),
+  });
+}
+
+describe('PagefindPlugin', () => {
+  test('collectIndexTargets only includes linked PDFs from reachable HTML pages', () => {
+    const htmlAssetsByPath = new Map([
+      [
+        'index.html',
+        '<a href="/about/">About</a><a href="/docs/guide.pdf">Guide</a>',
+      ],
+      ['about/index.html', '<p>About</p>'],
+      ['orphan/index.html', '<p>Orphan</p>'],
+    ]);
+    const pdfSourceByOutputPath = new Map([
+      ['/docs/guide.pdf', '/tmp/docs/guide.pdf'],
+      ['/docs/orphan.pdf', '/tmp/docs/orphan.pdf'],
+    ]);
+
+    const result = collectIndexTargets(
+      htmlAssetsByPath,
+      { basePath: '/' },
+      pdfSourceByOutputPath,
+    );
+
+    expect(result).toEqual({
+      reachableHtmlPaths: ['about/index.html', 'index.html'],
+      reachablePdfPaths: ['/docs/guide.pdf'],
+    });
+  });
+
+  test('buildIndex adds HTML files and per-page PDF custom records', async () => {
+    const calls = { htmlFiles: [], customRecords: [], outputPath: null };
+
+    await buildIndex({
+      distPath: '/tmp/dist',
+      htmlAssetsByPath: new Map([
+        ['index.html', '<html><body>Home</body></html>'],
+        ['about/index.html', '<html><body>About</body></html>'],
+      ]),
+      reachableHtmlPaths: ['index.html', 'about/index.html'],
+      reachablePdfPaths: ['/docs/guide.pdf'],
+      pdfSourceByOutputPath: new Map([
+        ['/docs/guide.pdf', '/tmp/docs/guide.pdf'],
+      ]),
+      loadPagefind: createFakePagefind(calls),
+      checkMutool: async () => {},
+      extractPages: async filePath => ({
+        pages: [
+          { pageNumber: 2, content: `EXTRACTED:${path.basename(filePath)}:2` },
+          { pageNumber: 5, content: `EXTRACTED:${path.basename(filePath)}:5` },
+        ],
+        hasExtractedText: true,
+      }),
+    });
+
+    expect(calls.htmlFiles).toEqual([
+      { sourcePath: 'index.html', content: '<html><body>Home</body></html>' },
+      {
+        sourcePath: 'about/index.html',
+        content: '<html><body>About</body></html>',
+      },
+    ]);
+    expect(calls.customRecords).toEqual([
+      {
+        url: '/docs/guide.pdf#page=2',
+        content: 'EXTRACTED:guide.pdf:2',
+        language: 'en',
+        meta: { title: 'guide.pdf', page: '2' },
+      },
+      {
+        url: '/docs/guide.pdf#page=5',
+        content: 'EXTRACTED:guide.pdf:5',
+        language: 'en',
+        meta: { title: 'guide.pdf', page: '5' },
+      },
+    ]);
+    expect(calls.outputPath).toBe('/tmp/dist/pagefind');
+    expect(calls.deleted).toBe(1);
+  });
+
+  test('buildIndex falls back to a single PDF record when text extraction is empty', async () => {
+    const calls = { customRecords: [] };
+
+    await buildIndex({
+      distPath: '/tmp/dist',
+      htmlAssetsByPath: new Map(),
+      reachableHtmlPaths: [],
+      reachablePdfPaths: ['/docs/guide.pdf'],
+      pdfSourceByOutputPath: new Map([
+        ['/docs/guide.pdf', '/tmp/docs/guide.pdf'],
+      ]),
+      loadPagefind: createFakePagefind(calls),
+      checkMutool: async () => {},
+      extractPages: async () => ({ pages: [], hasExtractedText: false }),
+    });
+
+    expect(calls.customRecords).toEqual([
+      {
+        url: '/docs/guide.pdf',
+        content: 'guide.pdf',
+        language: 'en',
+        meta: { title: 'guide.pdf' },
+      },
+    ]);
+  });
+});

package/webpack/pdf-text.js

@@ -0,0 +1,163 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawn } = require('child_process');
+
+let mutoolAvailabilityPromise = null;
+
+function formatMutoolCommand(args) {
+  return ['mutool', ...args].map(arg => JSON.stringify(arg)).join(' ');
+}
+
+function runMutool(args) {
+  return new Promise((resolve, reject) => {
+    const child = spawn('mutool', args, { stdio: ['ignore', 'pipe', 'pipe'] });
+
+    let settled = false;
+    let stdout = '';
+    let stderr = '';
+
+    child.stdout.setEncoding('utf8');
+    child.stdout.on('data', chunk => {
+      stdout += chunk;
+    });
+
+    child.stderr.setEncoding('utf8');
+    child.stderr.on('data', chunk => {
+      stderr += chunk;
+    });
+
+    child.on('error', err => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+
+      if (err && err.code === 'ENOENT') {
+        reject(
+          new Error(
+            'mutool is required for PDF text extraction but was not found on PATH',
+          ),
+        );
+        return;
+      }
+
+      reject(err);
+    });
+
+    child.on('close', code => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+
+      if (code !== 0) {
+        const output = stderr.trim() || stdout.trim();
+        const suffix = output.length > 0 ? `: ${output}` : '';
+        reject(
+          new Error(
+            `${formatMutoolCommand(args)} failed (code ${code})${suffix}`,
+          ),
+        );
+        return;
+      }
+
+      resolve({ stdout, stderr });
+    });
+  });
+}
+
+function assertMutoolAvailable() {
+  if (!mutoolAvailabilityPromise) {
+    mutoolAvailabilityPromise = runMutool(['-v']).then(() => undefined);
+  }
+
+  return mutoolAvailabilityPromise;
+}
+
+async function listNumberedPageFiles(dir, ext) {
+  const suffix = `.${ext}`;
+  const entries = await fs.promises.readdir(dir);
+
+  return entries
+    .filter(name => name.startsWith('page-') && name.endsWith(suffix))
+    .map(name => {
+      const pageNumText = name.slice(5, -suffix.length);
+      const pageNum = Number.parseInt(pageNumText, 10);
+      if (!Number.isInteger(pageNum)) {
+        return null;
+      }
+
+      return { fileName: name, filePath: path.join(dir, name), pageNum };
+    })
+    .filter(entry => entry !== null)
+    .sort((a, b) => a.pageNum - b.pageNum);
+}
+
+function normalizeExtractedText(text) {
+  return text.replace(/\s+/g, ' ').trim();
+}
+
+function buildPdfPageRecords(pageTexts) {
+  const pages = pageTexts
+    .map((text, i) => {
+      const normalized = normalizeExtractedText(text);
+      if (!normalized) {
+        return null;
+      }
+
+      return { pageNumber: i + 1, content: normalized };
+    })
+    .filter(Boolean);
+
+  return { pages, hasExtractedText: pages.length > 0 };
+}
+
+async function extractPdfPages(pdfPath) {
+  await assertMutoolAvailable();
+
+  const tempDir = await fs.promises.mkdtemp(
+    path.join(os.tmpdir(), 'pdf-text-'),
+  );
+  const textPattern = path.join(tempDir, 'page-%04d.txt');
+  let processingError = null;
+
+  try {
+    await runMutool(['draw', '-q', '-F', 'text', '-o', textPattern, pdfPath]);
+
+    const textFiles = await listNumberedPageFiles(tempDir, 'txt');
+    const pageTexts = await Promise.all(
+      textFiles.map(textFile =>
+        fs.promises.readFile(textFile.filePath, 'utf8'),
+      ),
+    );
+    const result = buildPdfPageRecords(pageTexts);
+
+    if (!result.hasExtractedText) {
+      console.warn(
+        `mutool did not extract searchable text for ${pdfPath}; indexing filename only`,
+      );
+    }
+
+    return result;
+  } catch (err) {
+    processingError = err;
+    throw err;
+  } finally {
+    try {
+      await fs.promises.rm(tempDir, { recursive: true, force: true });
+    } catch (cleanupErr) {
+      if (!processingError) {
+        console.warn(
+          `Failed to clean up temporary PDF extraction directory ${tempDir}: ${cleanupErr.message}`,
+        );
+      }
+    }
+  }
+}
+
+module.exports = {
+  assertMutoolAvailable,
+  buildPdfPageRecords,
+  extractPdfPages,
+};

package/webpack/print-flair-plugin.js

@@ -0,0 +1,22 @@
+const { getFlair } = require('./log');
+const { getDistDir } = require('./util');
+
+module.exports = {
+  apply: compiler => {
+    compiler.hooks.afterEmit.tap('AfterEmitPlugin', compilation => {
+      const isWatch =
+        !!compiler.watching ||
+        !!compiler.watchMode ||
+        !!compilation?.compiler?.watchMode;
+      if (isWatch) {
+        return;
+      }
+
+      const distDir = getDistDir();
+
+      console.log(getFlair());
+      console.log(`The build output is available at ${distDir}`);
+      console.log('Now use `tada serve` to start a local web server');
+    });
+  },
+};