@canopy-iiif/app 1.4.17 → 1.5.1

package/lib/build/build.js

@@ -29,6 +29,35 @@ const bibliography = require("../components/bibliography");
 let iiifRecordsCache = [];
 let pageRecords = [];
 
+function nowMs() {
+  try {
+    return Number(process.hrtime.bigint()) / 1e6;
+  } catch (_) {
+    return Date.now();
+  }
+}
+
+function formatDuration(ms) {
+  if (!Number.isFinite(ms) || ms < 0) return '0ms';
+  if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
+  return `${Math.round(ms)}ms`;
+}
+
+async function timeStage(label, store, fn) {
+  const start = nowMs();
+  try {
+    return await fn();
+  } finally {
+    const durationMs = nowMs() - start;
+    if (Array.isArray(store)) store.push({ label, durationMs });
+    try {
+      logLine(`⏱ ${label} completed in ${formatDuration(durationMs)}`, 'cyan', {
+        dim: true,
+      });
+    } catch (_) {}
+  }
+}
+
 async function ensureNoJekyllMarker() {
   try {
     await fsp.mkdir(OUT_DIR, { recursive: true });
@@ -43,113 +72,126 @@ async function build(options = {}) {
     console.error("No content directory found at", CONTENT_DIR);
     process.exit(1);
   }
+  const stageTimings = [];
+  const buildStart = nowMs();
 
   /**
    * Clean and prepare output directory
    */
-  logLine("\nClean and prepare directories", "magenta", {
-    bright: true,
-    underscore: true,
+  await timeStage("Clean and prepare directories", stageTimings, async () => {
+    logLine("\nClean and prepare directories", "magenta", {
+      bright: true,
+      underscore: true,
+    });
+    logLine("• Reset MDX cache", "blue", { dim: true });
+    mdx?.resetMdxCaches();
+    navigation?.resetNavigationCache?.();
+    referenced?.resetReferenceIndex?.();
+    bibliography?.resetBibliographyIndex?.();
+    if (!skipIiif) {
+      await cleanDir(OUT_DIR);
+      await ensureNoJekyllMarker();
+      logLine(`• Cleaned output directory`, "blue", { dim: true });
+    } else {
+      logLine("• Retaining cache (skipping IIIF rebuild)", "blue", { dim: true });
+    }
+    if (skipIiif) {
+      await ensureNoJekyllMarker();
+    }
   });
-  logLine("• Reset MDX cache", "blue", { dim: true });
-  mdx?.resetMdxCaches();
-  navigation?.resetNavigationCache?.();
-  referenced?.resetReferenceIndex?.();
-  bibliography?.resetBibliographyIndex?.();
-  if (!skipIiif) {
-    await cleanDir(OUT_DIR);
-    await ensureNoJekyllMarker();
-    logLine(`• Cleaned output directory`, "blue", { dim: true });
-  } else {
-    logLine("• Retaining cache (skipping IIIF rebuild)", "blue", { dim: true });
-  }
-  if (skipIiif) {
-    await ensureNoJekyllMarker();
-  }
 
   /**
    * Build IIIF Collection content from configured source(s).
    * This includes building IIIF manifests for works and collections,
    * as well as collecting search records for works.
    */
-  logLine("\nBuild IIIF Collection content", "magenta", {
-    bright: true,
-    underscore: true,
-  });
   let iiifRecords = [];
-  const CONFIG = await iiif.loadConfig();
-  if (!skipIiif) {
-    const results = await iiif.buildIiifCollectionPages(CONFIG);
-    iiifRecords = results?.iiifRecords;
-    iiifRecordsCache = Array.isArray(iiifRecords) ? iiifRecords : [];
-  } else {
-    iiifRecords = Array.isArray(iiifRecordsCache) ? iiifRecordsCache : [];
-    logLine(
-      `• Reusing cached IIIF search records (${iiifRecords.length})`,
-      "blue",
-      { dim: true }
-    );
-  }
-  // Ensure any configured featured manifests are cached (and thumbnails computed)
-  // so SSR interstitials can resolve items even if they are not part of
-  // the traversed collection or when IIIF build is skipped during incremental rebuilds.
-  try { await iiif.ensureFeaturedInCache(CONFIG); } catch (_) {}
-  try { await iiif.rebuildManifestIndexFromCache(); } catch (_) {}
+  let CONFIG;
+  await timeStage("Build IIIF Collection content", stageTimings, async () => {
+    logLine("\nBuild IIIF Collection content", "magenta", {
+      bright: true,
+      underscore: true,
+    });
+    CONFIG = await iiif.loadConfig();
+    if (!skipIiif) {
+      const results = await iiif.buildIiifCollectionPages(CONFIG);
+      iiifRecords = results?.iiifRecords;
+      iiifRecordsCache = Array.isArray(iiifRecords) ? iiifRecords : [];
+    } else {
+      iiifRecords = Array.isArray(iiifRecordsCache) ? iiifRecordsCache : [];
+      logLine(
+        `• Reusing cached IIIF search records (${iiifRecords.length})`,
+        "blue",
+        { dim: true }
+      );
+    }
+    // Ensure any configured featured manifests are cached (and thumbnails computed)
+    // so SSR interstitials can resolve items even if they are not part of
+    // the traversed collection or when IIIF build is skipped during incremental rebuilds.
+    try { await iiif.ensureFeaturedInCache(CONFIG); } catch (_) {}
+    try { await iiif.rebuildManifestIndexFromCache(); } catch (_) {}
+  });
 
   /**
    * Build contextual MDX content from the content directory.
    * This includes collecting page metadata for sitemap and search index,
    * as well as building all MDX pages to HTML.
    */
-  logLine("\nBuild contextual content from Markdown pages", "magenta", {
-    bright: true,
-    underscore: true,
+  await timeStage("Build contextual content from Markdown pages", stageTimings, async () => {
+    logLine("\nBuild contextual content from Markdown pages", "magenta", {
+      bright: true,
+      underscore: true,
+    });
+    // Interstitials read directly from the local IIIF cache; no API file needed
+    try {
+      bibliography?.buildBibliographyIndexSync?.();
+    } catch (err) {
+      logLine(
+        "• Failed to build bibliography index: " + String(err && err.message ? err.message : err),
+        "red",
+        { dim: true }
+      );
+    }
+    pageRecords = await searchBuild.collectMdxPageRecords();
+    await pages.buildContentTree(CONTENT_DIR, pageRecords);
+    logLine("✓ MDX pages built", "green");
   });
-  // Interstitials read directly from the local IIIF cache; no API file needed
-  try {
-    bibliography?.buildBibliographyIndexSync?.();
-  } catch (err) {
-    logLine(
-      "• Failed to build bibliography index: " + String(err && err.message ? err.message : err),
-      "red",
-      { dim: true }
-    );
-  }
-  pageRecords = await searchBuild.collectMdxPageRecords();
-  await pages.buildContentTree(CONTENT_DIR, pageRecords);
-  logLine("✓ MDX pages built", "green");
 
   /**
    * Build search index from IIIF and MDX records, then build or update
    * the search.html page and search runtime bundle.
    * This is done after all content is built so that the index is comprehensive.
    */
-  logLine("\nCreate search indices", "magenta", {
-    bright: true,
-    underscore: true,
+  await timeStage("Create search indices", stageTimings, async () => {
+    logLine("\nCreate search indices", "magenta", {
+      bright: true,
+      underscore: true,
+    });
+    try {
+      await ensureSearchInitialized();
+      const combined = await buildSearchIndex(iiifRecords, pageRecords);
+      await finalizeSearch(combined);
+    } catch (e) {
+      logLine("✗ Search index creation failed", "red", { bright: true });
+      logLine("  " + String(e), "red");
+    }
   });
-  try {
-    await ensureSearchInitialized();
-    const combined = await buildSearchIndex(iiifRecords, pageRecords);
-    await finalizeSearch(combined);
-  } catch (e) {
-    logLine("✗ Search index creation failed", "red", { bright: true });
-    logLine("  " + String(e), "red");
-  }
 
   /**
    * Generate a sitemap so static hosts can discover every rendered page.
    */
-  logLine("\nWrite sitemap", "magenta", {
-    bright: true,
-    underscore: true,
+  await timeStage("Write sitemap", stageTimings, async () => {
+    logLine("\nWrite sitemap", "magenta", {
+      bright: true,
+      underscore: true,
+    });
+    try {
+      await writeSitemap(iiifRecords, pageRecords);
+    } catch (e) {
+      logLine("✗ Failed to write sitemap.xml", "red", { bright: true });
+      logLine("  " + String(e), "red");
+    }
   });
-  try {
-    await writeSitemap(iiifRecords, pageRecords);
-  } catch (e) {
-    logLine("✗ Failed to write sitemap.xml", "red", { bright: true });
-    logLine("  " + String(e), "red");
-  }
 
   // No-op: Featured API file no longer written (SSR reads from cache directly)
 
@@ -157,24 +199,41 @@ async function build(options = {}) {
    * Prepare client runtimes (e.g. search) by bundling with esbuild.
    * This is done early so that MDX content can reference runtime assets if needed.
    */
-  logLine("\nPrepare client runtimes and stylesheets", "magenta", {
-    bright: true,
-    underscore: true,
+  await timeStage("Prepare client runtimes and stylesheets", stageTimings, async () => {
+    logLine("\nPrepare client runtimes and stylesheets", "magenta", {
+      bright: true,
+      underscore: true,
+    });
+    const tasks = [];
+    if (!process.env.CANOPY_SKIP_STYLES) {
+      tasks.push(
+        (async () => {
+          await ensureStyles();
+          logLine("✓ Wrote styles.css", "cyan");
+        })()
+      );
+    }
+    tasks.push(runtimes.prepareAllRuntimes());
+    await Promise.all(tasks);
   });
-  if (!process.env.CANOPY_SKIP_STYLES) {
-    await ensureStyles();
-    logLine("✓ Wrote styles.css", "cyan");
-  }
-  await runtimes.prepareAllRuntimes();
 
   /**
    * Copy static assets from the assets directory to the output directory.
    */
-  logLine("\nCopy static assets", "magenta", {
-    bright: true,
-    underscore: true,
+  await timeStage("Copy static assets", stageTimings, async () => {
+    logLine("\nCopy static assets", "magenta", {
+      bright: true,
+      underscore: true,
+    });
+    await copyAssets();
   });
-  await copyAssets();
+
+  const totalMs = nowMs() - buildStart;
+  logLine("\nBuild phase timings", "magenta", { bright: true, underscore: true });
+  for (const { label, durationMs } of stageTimings) {
+    logLine(`• ${label}: ${formatDuration(durationMs)}`, "blue", { dim: true });
+  }
+  logLine(`Total build time: ${formatDuration(totalMs)}`, "green", { bright: true });
 
 }
 

package/lib/build/iiif.js

@@ -47,8 +47,8 @@ const IIIF_CACHE_INDEX_MANIFESTS = path.join(
 );
 
 const DEFAULT_THUMBNAIL_SIZE = 400;
-const DEFAULT_CHUNK_SIZE = 20;
-const DEFAULT_FETCH_CONCURRENCY = 5;
+const DEFAULT_CHUNK_SIZE = 10;
+const DEFAULT_FETCH_CONCURRENCY = 1;
 const HERO_THUMBNAIL_SIZE = 800;
 const HERO_IMAGE_SIZES_ATTR = "(min-width: 1024px) 1280px, 100vw";
 const OG_IMAGE_WIDTH = 1200;
@@ -56,10 +56,22 @@ const OG_IMAGE_HEIGHT = 630;
 const HERO_REPRESENTATIVE_SIZE = Math.max(HERO_THUMBNAIL_SIZE, OG_IMAGE_WIDTH);
 const MAX_ENTRY_SLUG_LENGTH = 50;
 
-function resolvePositiveInteger(value, fallback) {
+function resolvePositiveInteger(value, fallback, options = {}) {
+  const allowZero = Boolean(options && options.allowZero);
   const num = Number(value);
-  if (Number.isFinite(num) && num > 0) return Math.max(1, Math.floor(num));
-  return Math.max(1, Math.floor(fallback));
+  if (Number.isFinite(num)) {
+    if (allowZero && num === 0) return 0;
+    if (num > 0) return Math.max(1, Math.floor(num));
+  }
+  const normalizedFallback = Number(fallback);
+  if (allowZero && normalizedFallback === 0) return 0;
+  return Math.max(1, Math.floor(normalizedFallback));
+}
+
+function formatDurationMs(ms) {
+  if (!Number.isFinite(ms) || ms < 0) return '0ms';
+  if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
+  return `${Math.round(ms)}ms`;
 }
 
 function resolveBoolean(value) {
@@ -699,6 +711,15 @@ const MEMO_ID_TO_SLUG = new Map();
 // collections/manifests share the same base title but mappings aren't yet saved.
 const RESERVED_SLUGS = {Manifest: new Set(), Collection: new Set()};
 
+function resetReservedSlugs() {
+  try {
+    Object.keys(RESERVED_SLUGS).forEach((key) => {
+      const set = RESERVED_SLUGS[key];
+      if (set && typeof set.clear === "function") set.clear();
+    });
+  } catch (_) {}
+}
+
 function computeUniqueSlug(index, baseSlug, id, type) {
   const byId = Array.isArray(index && index.byId) ? index.byId : [];
   const normId = normalizeIiifId(String(id || ""));
@@ -1411,6 +1432,11 @@ async function buildIiifCollectionPages(CONFIG) {
     DEFAULT_CHUNK_SIZE
   );
   const chunks = Math.ceil(tasks.length / chunkSize);
+  const requestedConcurrency = resolvePositiveInteger(
+    process.env.CANOPY_FETCH_CONCURRENCY,
+    DEFAULT_FETCH_CONCURRENCY,
+    {allowZero: true}
+  );
   // Summary before processing chunks
   try {
     const collectionsCount = visitedCollections.size || 0;
@@ -1419,6 +1445,11 @@ async function buildIiifCollectionPages(CONFIG) {
       "blue",
       {dim: true}
     );
+    const concurrencySummary =
+      requestedConcurrency === 0
+        ? "auto (no explicit cap)"
+        : String(requestedConcurrency);
+    logLine(`• Fetch concurrency: ${concurrencySummary}`, "blue", { dim: true });
   } catch (_) {}
   const iiifRecords = [];
   const navPlaceRecords = [];
@@ -1441,14 +1472,14 @@ async function buildIiifCollectionPages(CONFIG) {
 
   referenced.ensureReferenceIndex();
 
+  const chunkMetrics = [];
   for (let ci = 0; ci < chunks; ci++) {
     const chunk = tasks.slice(ci * chunkSize, (ci + 1) * chunkSize);
     logLine(`• Chunk ${ci + 1}/${chunks}`, "blue", {dim: true});
+    const chunkStart = Date.now();
 
-    const concurrency = resolvePositiveInteger(
-      process.env.CANOPY_FETCH_CONCURRENCY,
-      DEFAULT_FETCH_CONCURRENCY
-    );
+    const concurrency =
+      requestedConcurrency === 0 ? Math.max(1, chunk.length) : requestedConcurrency;
     let next = 0;
     const logs = new Array(chunk.length);
     let nextPrint = 0;
@@ -2102,6 +2133,42 @@ async function buildIiifCollectionPages(CONFIG) {
       () => worker()
     );
     await Promise.all(workers);
+    tryFlush();
+    const chunkDuration = Date.now() - chunkStart;
+    chunkMetrics.push({
+      index: ci + 1,
+      count: chunk.length,
+      durationMs: chunkDuration,
+      concurrency,
+    });
+    try {
+      const concurrencyLabel =
+        requestedConcurrency === 0 ? `${concurrency} (auto)` : String(concurrency);
+      logLine(
+        `⏱ Chunk ${ci + 1}/${chunks}: processed ${chunk.length} Manifest(s) in ${formatDurationMs(chunkDuration)} (concurrency ${concurrencyLabel})`,
+        "cyan",
+        { dim: true }
+      );
+    } catch (_) {}
+  }
+  if (chunkMetrics.length) {
+    const totalDuration = chunkMetrics.reduce(
+      (sum, entry) => sum + (entry.durationMs || 0),
+      0
+    );
+    const totalItems = chunkMetrics.reduce((sum, entry) => sum + (entry.count || 0), 0);
+    const avgDuration = chunkMetrics.length
+      ? totalDuration / chunkMetrics.length
+      : 0;
+    const rate = totalDuration > 0 ? totalItems / (totalDuration / 1000) : 0;
+    try {
+      const rateLabel = rate ? `${rate.toFixed(1)} manifest(s)/s` : 'n/a';
+      logLine(
+        `IIIF chunk summary: ${totalItems} Manifest(s) in ${formatDurationMs(totalDuration)} (avg chunk ${formatDurationMs(avgDuration)}, ${rateLabel})`,
+        "cyan",
+        { dim: true }
+      );
+    } catch (_) {}
   }
   try {
     await navPlace.writeNavPlaceDataset(navPlaceRecords);
@@ -2128,6 +2195,38 @@ module.exports = {
   rebuildManifestIndexFromCache,
 };
 
+// Expose a stable set of pure helper utilities for unit testing.
+module.exports.__TESTING__ = {
+  resolvePositiveInteger,
+  formatDurationMs,
+  resolveBoolean,
+  normalizeCollectionUris,
+  clampSlugLength,
+  isSlugTooLong,
+  normalizeSlugBase,
+  buildSlugWithSuffix,
+  normalizeStringList,
+  ensureThumbnailValue,
+  extractSummaryValues,
+  truncateSummary,
+  extractMetadataValues,
+  extractAnnotationText,
+  normalizeIiifId,
+  normalizeIiifType,
+  resolveParentFromPartOf,
+  computeUniqueSlug,
+  ensureBaseSlugFor,
+  resetReservedSlugs,
+  resolveThumbnailPreferences,
+  loadManifestIndex,
+  saveManifestIndex,
+  paths: {
+    IIIF_CACHE_INDEX,
+    IIIF_CACHE_INDEX_LEGACY,
+    IIIF_CACHE_INDEX_MANIFESTS,
+  },
+};
+
 // Debug: list collections cache after traversal
 try {
   if (process.env.CANOPY_IIIF_DEBUG === "1") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@canopy-iiif/app",
-  "version": "1.4.17",
+  "version": "1.5.1",
   "private": false,
   "license": "MIT",
   "author": "Mat Jordan <mat@northwestern.edu>",