@abreen/tada 1.0.2 → 1.1.0

package/{webpack/pagefind-plugin.test.js → build/pagefind.test.ts}

@@ -1,18 +1,36 @@
-const path = require('path');
-const { describe, expect, test } = require('bun:test');
-const { buildIndex, collectIndexTargets } = require('./pagefind-plugin');
+import path from 'path';
+import { describe, expect, test } from 'bun:test';
+import { buildIndex, collectIndexTargets } from './pagefind.js';
+import type { SiteVariables } from './types.js';
 
-function createFakePagefind(calls) {
+interface FakePagefindCalls {
+  htmlFiles?: { sourcePath: string; content: string }[];
+  customRecords?: {
+    url: string;
+    content: string;
+    language: string;
+    meta: Record<string, string>;
+  }[];
+  outputPath?: string | null;
+  deleted?: number;
+}
+
+function createFakePagefind(calls: FakePagefindCalls) {
   const fakeIndex = {
-    addHTMLFile: async file => {
+    addHTMLFile: async (file: { sourcePath: string; content: string }) => {
       calls.htmlFiles?.push(file);
       return { errors: [], file: { url: file.sourcePath, meta: {} } };
     },
-    addCustomRecord: async record => {
+    addCustomRecord: async (record: {
+      url: string;
+      content: string;
+      language: string;
+      meta: Record<string, string>;
+    }) => {
       calls.customRecords?.push(record);
       return { errors: [], file: { url: record.url, meta: record.meta } };
     },
-    writeFiles: async ({ outputPath }) => {
+    writeFiles: async ({ outputPath }: { outputPath: string }) => {
       calls.outputPath = outputPath;
       return { errors: [], outputPath };
     },
@@ -21,9 +39,10 @@ function createFakePagefind(calls) {
     },
   };
 
-  return async () => ({
-    createIndex: async () => ({ index: fakeIndex, errors: [] }),
-  });
+  return (() =>
+    Promise.resolve({
+      createIndex: async () => ({ index: fakeIndex, errors: [] }),
+    })) as unknown as () => Promise<typeof import('pagefind')>;
 }
 
 describe('PagefindPlugin', () => {
@@ -43,7 +62,7 @@ describe('PagefindPlugin', () => {
 
     const result = collectIndexTargets(
       htmlAssetsByPath,
-      { basePath: '/' },
+      { base: '', basePath: '/' } as SiteVariables,
       pdfSourceByOutputPath,
     );
 
@@ -54,7 +73,11 @@ describe('PagefindPlugin', () => {
   });
 
   test('buildIndex adds HTML files and per-page PDF custom records', async () => {
-    const calls = { htmlFiles: [], customRecords: [], outputPath: null };
+    const calls: FakePagefindCalls = {
+      htmlFiles: [],
+      customRecords: [],
+      outputPath: null,
+    };
 
     await buildIndex({
       distPath: '/tmp/dist',
@@ -103,8 +126,46 @@ describe('PagefindPlugin', () => {
     expect(calls.deleted).toBe(1);
   });
 
+  test('buildIndex prepends filename to page 1 content for searchability', async () => {
+    const calls: FakePagefindCalls = { htmlFiles: [], customRecords: [] };
+
+    await buildIndex({
+      distPath: '/tmp/dist',
+      htmlAssetsByPath: new Map(),
+      reachableHtmlPaths: [],
+      reachablePdfPaths: ['/docs/lecture1.pdf'],
+      pdfSourceByOutputPath: new Map([
+        ['/docs/lecture1.pdf', '/tmp/docs/lecture1.pdf'],
+      ]),
+      loadPagefind: createFakePagefind(calls),
+      checkMutool: async () => {},
+      extractPages: async () => ({
+        pages: [
+          { pageNumber: 1, content: 'Welcome to the course' },
+          { pageNumber: 2, content: 'Chapter one' },
+        ],
+        hasExtractedText: true,
+      }),
+    });
+
+    expect(calls.customRecords).toEqual([
+      {
+        url: '/docs/lecture1.pdf#page=1',
+        content: 'lecture1.pdf Welcome to the course',
+        language: 'en',
+        meta: { title: 'lecture1.pdf', page: '1' },
+      },
+      {
+        url: '/docs/lecture1.pdf#page=2',
+        content: 'Chapter one',
+        language: 'en',
+        meta: { title: 'lecture1.pdf', page: '2' },
+      },
+    ]);
+  });
+
   test('buildIndex falls back to a single PDF record when text extraction is empty', async () => {
-    const calls = { customRecords: [] };
+    const calls: FakePagefindCalls = { customRecords: [] };
 
     await buildIndex({
       distPath: '/tmp/dist',

package/build/pagefind.ts

@@ -0,0 +1,339 @@
+import path from 'path';
+import { makeLogger } from './log.js';
+import { collectReachableSiteAssets } from './reachability.js';
+import {
+  getContentDir,
+  getFilesByExtensions,
+  normalizeOutputPath,
+} from './util.js';
+import { assertMutoolAvailable, extractPdfPages } from './pdf-text.js';
+import type { SiteVariables } from './types.js';
+
+const log = makeLogger(__filename);
+const PAGEFIND_VERBOSE = process.env.TADA_LOG_LEVEL === 'debug';
+const PAGEFIND_OUTPUT_SUBDIR = 'pagefind';
+
+type PagefindModule = typeof import('pagefind');
+type PagefindIndex = Awaited<
+  ReturnType<PagefindModule['createIndex']>
+>['index'];
+
+let pagefindModulePromise: Promise<PagefindModule> | null = null;
+
+function getPagefind(): Promise<PagefindModule> {
+  if (!pagefindModulePromise) {
+    pagefindModulePromise = import('pagefind');
+  }
+  return pagefindModulePromise;
+}
+
+function formatPagefindErrors(
+  step: string,
+  errors: string[] | undefined,
+): string | null {
+  if (!errors?.length) {
+    return null;
+  }
+  return `${step} failed: ${errors.join(' | ')}`;
+}
+
+async function addHtmlFile(
+  index: NonNullable<PagefindIndex>,
+  htmlFile: { sourcePath: string; content: string },
+): Promise<void> {
+  const { errors: addErrors } = await index.addHTMLFile(htmlFile);
+  const addError = formatPagefindErrors(
+    `index.addHTMLFile(${htmlFile.sourcePath})`,
+    addErrors,
+  );
+  if (addError) {
+    throw new Error(addError);
+  }
+}
+
+async function addPdfRecord(
+  index: NonNullable<PagefindIndex>,
+  record: {
+    url: string;
+    content: string;
+    language: string;
+    meta: Record<string, string>;
+  },
+  sourcePath: string,
+): Promise<void> {
+  const { errors: addErrors } = await index.addCustomRecord(record);
+  const addError = formatPagefindErrors(
+    `index.addCustomRecord(${sourcePath})`,
+    addErrors,
+  );
+  if (addError) {
+    throw new Error(addError);
+  }
+}
+
+function getPdfSourceByOutputPath(): Map<string, string> {
+  const contentDir = getContentDir();
+  const pdfFiles: string[] = getFilesByExtensions(contentDir, ['pdf']);
+
+  return new Map(
+    pdfFiles.map((filePath: string) => {
+      const relPath = path.relative(contentDir, filePath);
+      return [normalizeOutputPath(`/${relPath}`), filePath] as const;
+    }),
+  );
+}
+
+interface IndexTargets {
+  reachableHtmlPaths: string[];
+  reachablePdfPaths: string[];
+}
+
+function collectIndexTargets(
+  htmlAssetsByPath: Map<string, string>,
+  siteVariables: SiteVariables,
+  pdfSourceByOutputPath: Map<string, string>,
+): IndexTargets {
+  if (htmlAssetsByPath.size === 0) {
+    return { reachableHtmlPaths: [], reachablePdfPaths: [] };
+  }
+
+  return collectReachableSiteAssets({
+    htmlAssetsByPath,
+    knownPdfPaths: new Set(pdfSourceByOutputPath.keys()),
+    rootPath: 'index.html',
+    basePath: siteVariables?.basePath || '/',
+  });
+}
+
+interface BuildIndexOptions {
+  distPath: string;
+  htmlAssetsByPath: Map<string, string>;
+  reachableHtmlPaths: string[];
+  reachablePdfPaths: string[];
+  pdfSourceByOutputPath: Map<string, string>;
+  loadPagefind?: () => Promise<PagefindModule>;
+  checkMutool?: () => Promise<void>;
+  extractPages?: typeof extractPdfPages;
+}
+
+async function buildIndex({
+  distPath,
+  htmlAssetsByPath,
+  reachableHtmlPaths,
+  reachablePdfPaths,
+  pdfSourceByOutputPath,
+  loadPagefind = getPagefind,
+  checkMutool = assertMutoolAvailable,
+  extractPages = extractPdfPages,
+}: BuildIndexOptions): Promise<void> {
+  const pagefind = await loadPagefind();
+  const { index, errors: createErrors } = await pagefind.createIndex({
+    keepIndexUrl: true,
+    verbose: PAGEFIND_VERBOSE,
+  });
+  const createError = formatPagefindErrors(
+    'pagefind.createIndex()',
+    createErrors,
+  );
+  if (createError) {
+    throw new Error(createError);
+  }
+  if (!index) {
+    throw new Error('pagefind.createIndex() did not return an index');
+  }
+
+  try {
+    for (const sourcePath of reachableHtmlPaths) {
+      await addHtmlFile(index, {
+        sourcePath,
+        content: htmlAssetsByPath.get(sourcePath)!,
+      });
+    }
+
+    let mutoolAvailable = true;
+    if (reachablePdfPaths.length > 0) {
+      try {
+        await checkMutool();
+      } catch {
+        mutoolAvailable = false;
+        log.warn`mutool was not found; search results will not include PDFs`;
+      }
+    }
+
+    for (const pdfPath of mutoolAvailable ? reachablePdfPaths : []) {
+      const sourceFilePath = pdfSourceByOutputPath.get(pdfPath);
+      if (!sourceFilePath) {
+        continue;
+      }
+
+      const { pages, hasExtractedText } = await extractPages(sourceFilePath);
+      const title = path.posix.basename(pdfPath);
+
+      if (!hasExtractedText) {
+        await addPdfRecord(
+          index,
+          { url: pdfPath, content: title, language: 'en', meta: { title } },
+          pdfPath,
+        );
+        continue;
+      }
+
+      for (const page of pages) {
+        const content =
+          page.pageNumber === 1 ? `${title} ${page.content}` : page.content;
+        await addPdfRecord(
+          index,
+          {
+            url: `${pdfPath}#page=${page.pageNumber}`,
+            content,
+            language: 'en',
+            meta: { title, page: String(page.pageNumber) },
+          },
+          `${pdfPath}#page=${page.pageNumber}`,
+        );
+      }
+    }
+
+    const { errors: writeErrors } = await index.writeFiles({
+      outputPath: path.join(distPath, PAGEFIND_OUTPUT_SUBDIR),
+    });
+    const writeError = formatPagefindErrors('index.writeFiles()', writeErrors);
+    if (writeError) {
+      throw new Error(writeError);
+    }
+  } finally {
+    await index.deleteIndex().catch(() => null);
+  }
+}
+
+interface RunPagefindOptions {
+  siteVariables: SiteVariables;
+  distPath: string;
+  htmlAssetsByPath: Map<string, string>;
+}
+
+export async function runPagefind({
+  siteVariables,
+  distPath,
+  htmlAssetsByPath,
+}: RunPagefindOptions): Promise<void> {
+  const pdfSourceByOutputPath = getPdfSourceByOutputPath();
+  const start = Date.now();
+
+  log.debug`Finding reachable pages for search index`;
+  const { reachableHtmlPaths, reachablePdfPaths } = collectIndexTargets(
+    htmlAssetsByPath,
+    siteVariables,
+    pdfSourceByOutputPath,
+  );
+
+  const snapshotReadyAt = Date.now();
+
+  let noun = reachableHtmlPaths.length === 1 ? 'page' : 'pages';
+  let message = `Building search index for ${reachableHtmlPaths.length} ${noun}`;
+  if (reachablePdfPaths.length > 0) {
+    noun = reachablePdfPaths.length === 1 ? 'PDF' : 'PDFs';
+    message += ` and ${reachablePdfPaths.length} ${noun}`;
+  }
+  log.info`${message}`;
+
+  await buildIndex({
+    distPath,
+    htmlAssetsByPath,
+    reachableHtmlPaths,
+    reachablePdfPaths,
+    pdfSourceByOutputPath,
+  });
+
+  try {
+    const pagefind = await getPagefind();
+    await pagefind.close();
+  } catch {
+    // Best-effort cleanup
+  }
+
+  const finishedAt = Date.now();
+  log.debug`Search index built in ${finishedAt - snapshotReadyAt}ms (${finishedAt - start}ms total)`;
+}
+
+export class WatchPagefindRunner {
+  private siteVariables: SiteVariables;
+  private watchRunInProgress: boolean;
+  private watchRunQueued: boolean;
+  private distPath: string | null;
+  private htmlCacheByAssetPath: Map<string, string>;
+
+  constructor(siteVariables: SiteVariables) {
+    this.siteVariables = siteVariables || {};
+    this.watchRunInProgress = false;
+    this.watchRunQueued = false;
+    this.distPath = null;
+    this.htmlCacheByAssetPath = new Map();
+  }
+
+  update(distPath: string, htmlAssetsByPath: Map<string, string>): void {
+    this.distPath = distPath;
+    this.htmlCacheByAssetPath = htmlAssetsByPath;
+  }
+
+  run(): void {
+    if (this.watchRunInProgress) {
+      this.watchRunQueued = true;
+      log.debug`Indexing is still running in the background; queueing a rerun`;
+      return;
+    }
+
+    this.watchRunInProgress = true;
+    this.watchRunQueued = false;
+    const distPath = this.distPath!;
+    const htmlAssetsByPath = new Map(this.htmlCacheByAssetPath);
+    const pdfSourceByOutputPath = getPdfSourceByOutputPath();
+    const start = Date.now();
+
+    log.debug`Preparing search index background snapshot`;
+
+    let reachableHtmlPaths: string[];
+    let reachablePdfPaths: string[];
+    try {
+      ({ reachableHtmlPaths, reachablePdfPaths } = collectIndexTargets(
+        htmlAssetsByPath,
+        this.siteVariables,
+        pdfSourceByOutputPath,
+      ));
+    } catch (err) {
+      this.watchRunInProgress = false;
+      log.warn`Pagefind failed: ${(err as Error).message}`;
+      if (this.watchRunQueued) {
+        this.run();
+      }
+      return;
+    }
+
+    const snapshotReadyAt = Date.now();
+    log.debug`Building search index in background`;
+    buildIndex({
+      distPath,
+      htmlAssetsByPath,
+      reachableHtmlPaths,
+      reachablePdfPaths,
+      pdfSourceByOutputPath,
+    })
+      .then(() => {
+        const finishedAt = Date.now();
+        log.debug`Search index ready after ${finishedAt - snapshotReadyAt}ms (${finishedAt - start}ms total)`;
+      })
+      .catch(err => {
+        const failedAt = Date.now();
+        log.warn`Search index failed after ${failedAt - snapshotReadyAt}ms of indexing (${failedAt - start}ms total): ${err.message}`;
+      })
+      .finally(() => {
+        this.watchRunInProgress = false;
+        if (this.watchRunQueued) {
+          log.debug`Starting queued Pagefind background rerun`;
+          this.run();
+        }
+      });
+  }
+}
+
+export { buildIndex, collectIndexTargets };

package/{webpack/pdf-text.js → build/pdf-text.ts}

@@ -1,15 +1,30 @@
-const fs = require('fs');
-const os = require('os');
-const path = require('path');
-const { spawn } = require('child_process');
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { spawn } from 'child_process';
+
+interface MutoolOutput {
+  stdout: string;
+  stderr: string;
+}
+
+export interface PdfPage {
+  pageNumber: number;
+  content: string;
+}
+
+export interface PdfExtractResult {
+  pages: PdfPage[];
+  hasExtractedText: boolean;
+}
 
-let mutoolAvailabilityPromise = null;
+let mutoolAvailabilityPromise: Promise<void> | null = null;
 
-function formatMutoolCommand(args) {
+function formatMutoolCommand(args: string[]): string {
   return ['mutool', ...args].map(arg => JSON.stringify(arg)).join(' ');
 }
 
-function runMutool(args) {
+function runMutool(args: string[]): Promise<MutoolOutput> {
   return new Promise((resolve, reject) => {
     const child = spawn('mutool', args, { stdio: ['ignore', 'pipe', 'pipe'] });
 
@@ -18,16 +33,16 @@ function runMutool(args) {
     let stderr = '';
 
     child.stdout.setEncoding('utf8');
-    child.stdout.on('data', chunk => {
+    child.stdout.on('data', (chunk: string) => {
       stdout += chunk;
     });
 
     child.stderr.setEncoding('utf8');
-    child.stderr.on('data', chunk => {
+    child.stderr.on('data', (chunk: string) => {
       stderr += chunk;
     });
 
-    child.on('error', err => {
+    child.on('error', (err: NodeJS.ErrnoException) => {
       if (settled) {
         return;
       }
@@ -45,7 +60,7 @@ function runMutool(args) {
       reject(err);
     });
 
-    child.on('close', code => {
+    child.on('close', (code: number | null) => {
       if (settled) {
         return;
       }
@@ -67,7 +82,7 @@ function runMutool(args) {
   });
 }
 
-function assertMutoolAvailable() {
+export function assertMutoolAvailable(): Promise<void> {
   if (!mutoolAvailabilityPromise) {
     mutoolAvailabilityPromise = runMutool(['-v']).then(() => undefined);
   }
@@ -75,7 +90,16 @@ function assertMutoolAvailable() {
   return mutoolAvailabilityPromise;
 }
 
-async function listNumberedPageFiles(dir, ext) {
+interface NumberedPageFile {
+  fileName: string;
+  filePath: string;
+  pageNum: number;
+}
+
+async function listNumberedPageFiles(
+  dir: string,
+  ext: string,
+): Promise<NumberedPageFile[]> {
   const suffix = `.${ext}`;
   const entries = await fs.promises.readdir(dir);
 
@@ -90,16 +114,16 @@ async function listNumberedPageFiles(dir, ext) {
 
       return { fileName: name, filePath: path.join(dir, name), pageNum };
     })
-    .filter(entry => entry !== null)
+    .filter((entry): entry is NumberedPageFile => entry !== null)
     .sort((a, b) => a.pageNum - b.pageNum);
 }
 
-function normalizeExtractedText(text) {
+function normalizeExtractedText(text: string): string {
   return text.replace(/\s+/g, ' ').trim();
 }
 
-function buildPdfPageRecords(pageTexts) {
-  const pages = pageTexts
+export function buildPdfPageRecords(pageTexts: string[]): PdfExtractResult {
+  const pages: PdfPage[] = pageTexts
     .map((text, i) => {
       const normalized = normalizeExtractedText(text);
       if (!normalized) {
@@ -108,19 +132,21 @@ function buildPdfPageRecords(pageTexts) {
 
       return { pageNumber: i + 1, content: normalized };
     })
-    .filter(Boolean);
+    .filter((entry): entry is PdfPage => entry !== null);
 
   return { pages, hasExtractedText: pages.length > 0 };
 }
 
-async function extractPdfPages(pdfPath) {
+export async function extractPdfPages(
+  pdfPath: string,
+): Promise<PdfExtractResult> {
   await assertMutoolAvailable();
 
   const tempDir = await fs.promises.mkdtemp(
     path.join(os.tmpdir(), 'pdf-text-'),
   );
   const textPattern = path.join(tempDir, 'page-%04d.txt');
-  let processingError = null;
+  let processingError: unknown = null;
 
   try {
     await runMutool(['draw', '-q', '-F', 'text', '-o', textPattern, pdfPath]);
@@ -149,15 +175,9 @@ async function extractPdfPages(pdfPath) {
     } catch (cleanupErr) {
       if (!processingError) {
         console.warn(
-          `Failed to clean up temporary PDF extraction directory ${tempDir}: ${cleanupErr.message}`,
+          `Failed to clean up temporary PDF extraction directory ${tempDir}: ${(cleanupErr as Error).message}`,
         );
       }
     }
   }
 }
-
-module.exports = {
-  assertMutoolAvailable,
-  buildPdfPageRecords,
-  extractPdfPages,
-};