@canopy-iiif/app 0.6.28 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/build/assets.js +30 -0
- package/lib/build/build.js +128 -0
- package/lib/{dev.js → build/dev.js} +494 -205
- package/lib/build/facets.js +19 -0
- package/lib/{iiif.js → build/iiif.js} +242 -384
- package/lib/build/log.js +31 -0
- package/lib/{mdx.js → build/mdx.js} +13 -10
- package/lib/build/pages.js +141 -0
- package/lib/build/runtimes.js +58 -0
- package/lib/build/search-index.js +42 -0
- package/lib/build/search-workflow.js +61 -0
- package/lib/build/search.js +219 -0
- package/lib/build/styles.js +141 -0
- package/lib/{thumbnail.js → iiif/thumbnail.js} +0 -1
- package/lib/index.js +2 -3
- package/lib/{search.js → search/search.js} +8 -8
- package/package.json +2 -1
- package/lib/build.js +0 -762
- package/lib/components/IIIFCard.js +0 -102
- package/lib/log.js +0 -64
- package/lib/runtime/command-entry.jsx +0 -44
- /package/lib/{devtoast.config.json → build/devtoast.config.json} +0 -0
- /package/lib/{devtoast.css → build/devtoast.css} +0 -0
- /package/lib/{search-app.jsx → search/search-app.jsx} +0 -0
|
@@ -11,7 +11,7 @@ const {
|
|
|
11
11
|
CONTENT_DIR,
|
|
12
12
|
ensureDirSync,
|
|
13
13
|
htmlShell,
|
|
14
|
-
} = require("
|
|
14
|
+
} = require("../common");
|
|
15
15
|
const mdx = require("./mdx");
|
|
16
16
|
const { log, logLine, logResponse } = require("./log");
|
|
17
17
|
|
|
@@ -88,13 +88,12 @@ async function readJsonFromUri(uri, options = { log: false }) {
|
|
|
88
88
|
if (options && options.log) {
|
|
89
89
|
try {
|
|
90
90
|
if (res && res.ok) {
|
|
91
|
-
|
|
92
|
-
logLine(`✓ ${String(uri)} ➜ ${res.status}`, "yellow", {
|
|
91
|
+
logLine(`↓ ${String(uri)} → ${res.status}`, "yellow", {
|
|
93
92
|
bright: true,
|
|
94
93
|
});
|
|
95
94
|
} else {
|
|
96
95
|
const code = res ? res.status : "ERR";
|
|
97
|
-
logLine(
|
|
96
|
+
logLine(`⊘ ${String(uri)} → ${code}`, "red", { bright: true });
|
|
98
97
|
}
|
|
99
98
|
} catch (_) {}
|
|
100
99
|
}
|
|
@@ -247,13 +246,18 @@ function computeUniqueSlug(index, baseSlug, id, type) {
|
|
|
247
246
|
function ensureBaseSlugFor(index, baseSlug, id, type) {
|
|
248
247
|
try {
|
|
249
248
|
const byId = Array.isArray(index && index.byId) ? index.byId : [];
|
|
250
|
-
const normId = normalizeIiifId(String(id ||
|
|
249
|
+
const normId = normalizeIiifId(String(id || ""));
|
|
251
250
|
const existingWithBase = byId.find(
|
|
252
251
|
(e) => e && e.type === type && String(e.slug) === String(baseSlug)
|
|
253
252
|
);
|
|
254
253
|
if (existingWithBase && normalizeIiifId(existingWithBase.id) !== normId) {
|
|
255
254
|
// Reassign the existing entry to the next available suffix to free the base
|
|
256
|
-
const newSlug = computeUniqueSlug(
|
|
255
|
+
const newSlug = computeUniqueSlug(
|
|
256
|
+
index,
|
|
257
|
+
baseSlug,
|
|
258
|
+
existingWithBase.id,
|
|
259
|
+
type
|
|
260
|
+
);
|
|
257
261
|
if (newSlug && newSlug !== baseSlug) existingWithBase.slug = newSlug;
|
|
258
262
|
}
|
|
259
263
|
} catch (_) {}
|
|
@@ -455,7 +459,7 @@ async function saveCachedCollection(collection, id, parentId) {
|
|
|
455
459
|
try {
|
|
456
460
|
if (process.env.CANOPY_IIIF_DEBUG === "1") {
|
|
457
461
|
const { logLine } = require("./log");
|
|
458
|
-
logLine(`IIIF: saved collection
|
|
462
|
+
logLine(`IIIF: saved collection → ${slug}.json`, "cyan", { dim: true });
|
|
459
463
|
}
|
|
460
464
|
} catch (_) {}
|
|
461
465
|
index.byId = Array.isArray(index.byId) ? index.byId : [];
|
|
@@ -476,351 +480,161 @@ async function saveCachedCollection(collection, id, parentId) {
|
|
|
476
480
|
}
|
|
477
481
|
|
|
478
482
|
async function loadConfig() {
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
const configPath = path.resolve(overrideConfigPath || "canopy.yml");
|
|
488
|
-
if (fs.existsSync(configPath)) {
|
|
489
|
-
try {
|
|
490
|
-
const raw = await fsp.readFile(configPath, "utf8");
|
|
491
|
-
const data = yaml.load(raw) || {};
|
|
492
|
-
const d = data || {};
|
|
493
|
-
const di = d.iiif || {};
|
|
494
|
-
CONFIG = {
|
|
495
|
-
collection: {
|
|
496
|
-
uri: (d.collection && d.collection.uri) || CONFIG.collection.uri,
|
|
497
|
-
},
|
|
498
|
-
iiif: {
|
|
499
|
-
chunkSize:
|
|
500
|
-
Number(di.chunkSize || CONFIG.iiif.chunkSize) ||
|
|
501
|
-
CONFIG.iiif.chunkSize,
|
|
502
|
-
concurrency:
|
|
503
|
-
Number(di.concurrency || CONFIG.iiif.concurrency) ||
|
|
504
|
-
CONFIG.iiif.concurrency,
|
|
505
|
-
thumbnails: {
|
|
506
|
-
unsafe: !!(di.thumbnails && di.thumbnails.unsafe === true),
|
|
507
|
-
preferredSize:
|
|
508
|
-
Number(di.thumbnails && di.thumbnails.preferredSize) || 1200,
|
|
509
|
-
},
|
|
510
|
-
},
|
|
511
|
-
metadata: Array.isArray(d.metadata)
|
|
512
|
-
? d.metadata.map((s) => String(s)).filter(Boolean)
|
|
513
|
-
: [],
|
|
514
|
-
};
|
|
515
|
-
const src = overrideConfigPath ? overrideConfigPath : "canopy.yml";
|
|
516
|
-
logLine(`[canopy] Loaded config from ${src}...`, "white");
|
|
517
|
-
} catch (e) {
|
|
518
|
-
console.warn(
|
|
519
|
-
"[canopy] Failed to read",
|
|
520
|
-
overrideConfigPath ? overrideConfigPath : "canopy.yml",
|
|
521
|
-
e.message
|
|
522
|
-
);
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
if (process.env.CANOPY_COLLECTION_URI) {
|
|
526
|
-
CONFIG.collection.uri = String(process.env.CANOPY_COLLECTION_URI);
|
|
527
|
-
console.log("Using collection URI from CANOPY_COLLECTION_URI");
|
|
483
|
+
const cfgPath = path.resolve("canopy.yml");
|
|
484
|
+
if (!fs.existsSync(cfgPath)) return {};
|
|
485
|
+
const raw = await fsp.readFile(cfgPath, "utf8");
|
|
486
|
+
let cfg = {};
|
|
487
|
+
try {
|
|
488
|
+
cfg = yaml.load(raw) || {};
|
|
489
|
+
} catch (_) {
|
|
490
|
+
cfg = {};
|
|
528
491
|
}
|
|
529
|
-
return
|
|
492
|
+
return cfg || {};
|
|
530
493
|
}
|
|
531
494
|
|
|
495
|
+
// Traverse IIIF collection, cache manifests/collections, and render pages
|
|
532
496
|
async function buildIiifCollectionPages(CONFIG) {
|
|
533
|
-
const
|
|
534
|
-
const worksLayoutPath = path.join(worksDir, "_layout.mdx");
|
|
535
|
-
if (!fs.existsSync(worksLayoutPath)) {
|
|
536
|
-
console.log(
|
|
537
|
-
"IIIF: No content/works/_layout.mdx found; skipping IIIF page build."
|
|
538
|
-
);
|
|
539
|
-
return { searchRecords: [] };
|
|
540
|
-
}
|
|
541
|
-
ensureDirSync(IIIF_CACHE_MANIFESTS_DIR);
|
|
542
|
-
ensureDirSync(IIIF_CACHE_COLLECTIONS_DIR);
|
|
543
|
-
// Debug: list current collections cache contents
|
|
544
|
-
try {
|
|
545
|
-
if (process.env.CANOPY_IIIF_DEBUG === "1") {
|
|
546
|
-
const { logLine } = require("./log");
|
|
547
|
-
try {
|
|
548
|
-
const files = fs.existsSync(IIIF_CACHE_COLLECTIONS_DIR)
|
|
549
|
-
? fs
|
|
550
|
-
.readdirSync(IIIF_CACHE_COLLECTIONS_DIR)
|
|
551
|
-
.filter((n) => /\.json$/i.test(n))
|
|
552
|
-
: [];
|
|
553
|
-
const head = files.slice(0, 8).join(", ");
|
|
554
|
-
logLine(
|
|
555
|
-
`IIIF: cache/collections (start): ${files.length} file(s)` +
|
|
556
|
-
(head ? ` [${head}${files.length > 8 ? ", …" : ""}]` : ""),
|
|
557
|
-
"blue",
|
|
558
|
-
{ dim: true }
|
|
559
|
-
);
|
|
560
|
-
} catch (_) {}
|
|
561
|
-
}
|
|
562
|
-
} catch (_) {}
|
|
497
|
+
const cfg = CONFIG || (await loadConfig());
|
|
563
498
|
const collectionUri =
|
|
564
|
-
(
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
} catch (_) {}
|
|
575
|
-
// Decide cache policy/log before any fetch
|
|
576
|
-
let index = await loadManifestIndex();
|
|
577
|
-
const prevUri = (index && index.collection && index.collection.uri) || "";
|
|
578
|
-
const uriChanged = !!(collectionUri && prevUri && prevUri !== collectionUri);
|
|
579
|
-
if (uriChanged) {
|
|
580
|
-
try {
|
|
581
|
-
const { logLine } = require("./log");
|
|
582
|
-
logLine("IIIF Collection changed. Resetting cache.\n", "cyan");
|
|
583
|
-
} catch (_) {}
|
|
584
|
-
await flushManifestCache();
|
|
585
|
-
index.byId = [];
|
|
586
|
-
} else {
|
|
587
|
-
try {
|
|
588
|
-
const { logLine } = require("./log");
|
|
589
|
-
logLine(
|
|
590
|
-
"IIIF Collection unchanged. Preserving cache. Detecting cached resources...\n",
|
|
591
|
-
"cyan"
|
|
592
|
-
);
|
|
593
|
-
} catch (_) {}
|
|
594
|
-
}
|
|
595
|
-
let collection = null;
|
|
596
|
-
if (collectionUri) {
|
|
597
|
-
// Try cached root collection first
|
|
598
|
-
collection = await loadCachedCollectionById(collectionUri);
|
|
599
|
-
if (collection) {
|
|
600
|
-
try {
|
|
601
|
-
const { logLine } = require("./log");
|
|
602
|
-
logLine(`✓ ${String(collectionUri)} ➜ Cached`, "yellow");
|
|
603
|
-
} catch (_) {}
|
|
604
|
-
} else {
|
|
605
|
-
collection = await readJsonFromUri(collectionUri, { log: true });
|
|
606
|
-
if (collection) {
|
|
607
|
-
try {
|
|
608
|
-
await saveCachedCollection(
|
|
609
|
-
collection,
|
|
610
|
-
String(collection.id || collection["@id"] || collectionUri),
|
|
611
|
-
""
|
|
612
|
-
);
|
|
613
|
-
} catch (_) {}
|
|
614
|
-
}
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
if (!collection) {
|
|
618
|
-
console.warn("IIIF: No collection available; skipping.");
|
|
619
|
-
// Still write/update global index with configured URI, so other steps can rely on it
|
|
620
|
-
try {
|
|
621
|
-
const index = await loadManifestIndex();
|
|
622
|
-
index.collection = {
|
|
623
|
-
uri: String(collectionUri || ""),
|
|
624
|
-
hash: "",
|
|
625
|
-
updatedAt: new Date().toISOString(),
|
|
626
|
-
};
|
|
627
|
-
await saveManifestIndex(index);
|
|
628
|
-
} catch (_) {}
|
|
499
|
+
(cfg && cfg.collection && cfg.collection.uri) ||
|
|
500
|
+
process.env.CANOPY_COLLECTION_URI ||
|
|
501
|
+
"";
|
|
502
|
+
if (!collectionUri) return { searchRecords: [] };
|
|
503
|
+
|
|
504
|
+
// Fetch top-level collection
|
|
505
|
+
logLine("• Traversing IIIF Collection(s)", "blue", { dim: true });
|
|
506
|
+
const root = await readJsonFromUri(collectionUri, { log: true });
|
|
507
|
+
if (!root) {
|
|
508
|
+
logLine("IIIF: Failed to fetch collection", "red");
|
|
629
509
|
return { searchRecords: [] };
|
|
630
510
|
}
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
const currentSig = {
|
|
634
|
-
uri: String(collectionUri || ""),
|
|
635
|
-
hash: computeHash(collectionForHash),
|
|
636
|
-
};
|
|
637
|
-
index.collection = { ...currentSig, updatedAt: new Date().toISOString() };
|
|
638
|
-
// Upsert root collection entry in index.byId
|
|
511
|
+
const normalizedRoot = await normalizeToV3(root);
|
|
512
|
+
// Save collection cache
|
|
639
513
|
try {
|
|
640
|
-
|
|
641
|
-
|
|
514
|
+
await saveCachedCollection(
|
|
515
|
+
normalizedRoot,
|
|
516
|
+
normalizedRoot.id || collectionUri,
|
|
517
|
+
""
|
|
642
518
|
);
|
|
643
|
-
const title = firstLabelString(collection && collection.label);
|
|
644
|
-
const baseSlug = slugify(title || "collection", { lower: true, strict: true, trim: true }) || "collection";
|
|
645
|
-
index.byId = Array.isArray(index.byId) ? index.byId : [];
|
|
646
|
-
const slug = ensureBaseSlugFor(index, baseSlug, rootId, 'Collection');
|
|
647
|
-
const existingIdx = index.byId.findIndex(
|
|
648
|
-
(e) => e && normalizeIiifId(e.id) === rootId && e.type === "Collection"
|
|
649
|
-
);
|
|
650
|
-
const entry = { id: rootId, type: "Collection", slug, parent: "" };
|
|
651
|
-
if (existingIdx >= 0) index.byId[existingIdx] = entry;
|
|
652
|
-
else index.byId.push(entry);
|
|
653
|
-
try { (RESERVED_SLUGS && RESERVED_SLUGS.Collection || new Set()).add(slug); } catch (_) {}
|
|
654
519
|
} catch (_) {}
|
|
655
|
-
await saveManifestIndex(index);
|
|
656
520
|
|
|
657
|
-
|
|
658
|
-
// Recursively collect manifests across subcollections
|
|
521
|
+
// Recursively traverse Collections and gather all Manifest tasks
|
|
659
522
|
const tasks = [];
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
if (c) {
|
|
663
|
-
try {
|
|
664
|
-
// Emit a standard Cached line so it mirrors network logs
|
|
665
|
-
const { logLine } = require("./log");
|
|
666
|
-
logLine(`✓ ${String(id)} ➜ Cached`, "yellow");
|
|
667
|
-
} catch (_) {}
|
|
668
|
-
if (lns) lns.push([`✓ ${String(id)} ➜ Cached`, "yellow"]);
|
|
669
|
-
// Always normalize to Presentation 3 for consistent traversal
|
|
670
|
-
return await normalizeToV3(c);
|
|
671
|
-
}
|
|
672
|
-
if (!fetchAllowed) return null;
|
|
673
|
-
const fetched = await readJsonFromUri(String(id), { log: true });
|
|
674
|
-
const normalized = await normalizeToV3(fetched);
|
|
523
|
+
const visitedCollections = new Set(); // normalized ids
|
|
524
|
+
const norm = (x) => {
|
|
675
525
|
try {
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
""
|
|
680
|
-
);
|
|
681
|
-
} catch (_) {}
|
|
682
|
-
return normalized || fetched;
|
|
683
|
-
}
|
|
684
|
-
async function collectTasksFromCollection(colObj, parentUri, visited) {
|
|
685
|
-
if (!colObj) return;
|
|
686
|
-
// Prefer the URI we traversed from (parentUri) to work around sources
|
|
687
|
-
// that report the same id for multiple pages (e.g., Internet Archive).
|
|
688
|
-
const colId = String(parentUri || colObj.id || colObj["@id"] || "");
|
|
689
|
-
visited = visited || new Set();
|
|
690
|
-
if (colId) {
|
|
691
|
-
if (visited.has(colId)) return;
|
|
692
|
-
visited.add(colId);
|
|
526
|
+
return normalizeIiifId(String(x || ""));
|
|
527
|
+
} catch (_) {
|
|
528
|
+
return String(x || "");
|
|
693
529
|
}
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
const
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
slug,
|
|
725
|
-
parent: normalizeIiifId(String(colId || parentUri || "")),
|
|
726
|
-
};
|
|
727
|
-
const existing = idx.byId.findIndex(
|
|
728
|
-
(e) =>
|
|
729
|
-
e &&
|
|
730
|
-
normalizeIiifId(e.id) === subIdNorm &&
|
|
731
|
-
e.type === "Collection"
|
|
732
|
-
);
|
|
733
|
-
if (existing >= 0) idx.byId[existing] = entry;
|
|
734
|
-
else idx.byId.push(entry);
|
|
735
|
-
await saveManifestIndex(idx);
|
|
736
|
-
} catch (_) {}
|
|
737
|
-
await collectTasksFromCollection(subRaw, String(id), visited);
|
|
738
|
-
} else if (/^https?:\/\//i.test(String(id || ""))) {
|
|
739
|
-
let norm = await loadOrFetchCollectionById(String(id));
|
|
740
|
-
const nt = String((norm && (norm.type || norm["@type"])) || "");
|
|
741
|
-
if (nt.includes("Collection")) {
|
|
742
|
-
try {
|
|
743
|
-
const title = firstLabelString(norm && norm.label);
|
|
744
|
-
const baseSlug = slugify(title || "collection", { lower: true, strict: true, trim: true }) || "collection";
|
|
745
|
-
const idx = await loadManifestIndex();
|
|
746
|
-
idx.byId = Array.isArray(idx.byId) ? idx.byId : [];
|
|
747
|
-
const normId = normalizeIiifId(String(id));
|
|
748
|
-
const slug = computeUniqueSlug(idx, baseSlug, normId, 'Collection');
|
|
749
|
-
const entry = {
|
|
750
|
-
id: normId,
|
|
751
|
-
type: "Collection",
|
|
752
|
-
slug,
|
|
753
|
-
parent: normalizeIiifId(String(colId || parentUri || "")),
|
|
754
|
-
};
|
|
755
|
-
const existing = idx.byId.findIndex(
|
|
756
|
-
(e) => e && normalizeIiifId(e.id) === normId && e.type === "Collection"
|
|
757
|
-
);
|
|
758
|
-
if (existing >= 0) idx.byId[existing] = entry;
|
|
759
|
-
else idx.byId.push(entry);
|
|
760
|
-
await saveManifestIndex(idx);
|
|
761
|
-
} catch (_) {}
|
|
762
|
-
try {
|
|
763
|
-
await saveCachedCollection(
|
|
764
|
-
norm,
|
|
765
|
-
String(id),
|
|
766
|
-
String(colId || parentUri || "")
|
|
767
|
-
);
|
|
768
|
-
} catch (_) {}
|
|
769
|
-
await collectTasksFromCollection(norm, String(id), visited);
|
|
770
|
-
} else if (nt.includes("Manifest")) {
|
|
771
|
-
tasks.push({
|
|
772
|
-
id: String(id),
|
|
773
|
-
parent: String(colId || parentUri || ""),
|
|
774
|
-
});
|
|
530
|
+
};
|
|
531
|
+
async function gatherFromCollection(colLike, parentId) {
|
|
532
|
+
try {
|
|
533
|
+
// Resolve uri/id
|
|
534
|
+
const uri =
|
|
535
|
+
typeof colLike === "string"
|
|
536
|
+
? colLike
|
|
537
|
+
: (colLike && (colLike.id || colLike["@id"])) || "";
|
|
538
|
+
const col =
|
|
539
|
+
typeof colLike === "object" && colLike && colLike.items
|
|
540
|
+
? colLike
|
|
541
|
+
: await readJsonFromUri(uri, { log: true });
|
|
542
|
+
if (!col) return;
|
|
543
|
+
const ncol = await normalizeToV3(col);
|
|
544
|
+
const colId = String((ncol && (ncol.id || uri)) || uri);
|
|
545
|
+
const nid = norm(colId);
|
|
546
|
+
if (visitedCollections.has(nid)) return; // avoid cycles
|
|
547
|
+
visitedCollections.add(nid);
|
|
548
|
+
try {
|
|
549
|
+
await saveCachedCollection(ncol, colId, parentId || "");
|
|
550
|
+
} catch (_) {}
|
|
551
|
+
const itemsArr = Array.isArray(ncol.items) ? ncol.items : [];
|
|
552
|
+
for (const entry of itemsArr) {
|
|
553
|
+
if (!entry) continue;
|
|
554
|
+
const t = String(entry.type || entry["@type"] || "").toLowerCase();
|
|
555
|
+
const entryId = entry.id || entry["@id"] || "";
|
|
556
|
+
if (t === "manifest") {
|
|
557
|
+
tasks.push({ id: entryId, parent: colId });
|
|
558
|
+
} else if (t === "collection") {
|
|
559
|
+
await gatherFromCollection(entryId, colId);
|
|
775
560
|
}
|
|
776
561
|
}
|
|
777
|
-
|
|
562
|
+
// Traverse strictly by parent/child hierarchy (Presentation 3): items → Manifest or Collection
|
|
563
|
+
} catch (_) {}
|
|
778
564
|
}
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
String((collectionV3 && (collectionV3.id || collectionV3["@id"])) || collectionUri || ""),
|
|
784
|
-
new Set()
|
|
785
|
-
);
|
|
565
|
+
await gatherFromCollection(normalizedRoot, "");
|
|
566
|
+
if (!tasks.length) return { searchRecords: [] };
|
|
567
|
+
|
|
568
|
+
// Split into chunks and process with limited concurrency
|
|
786
569
|
const chunkSize = Math.max(
|
|
787
570
|
1,
|
|
788
571
|
Number(
|
|
789
|
-
process.env.CANOPY_CHUNK_SIZE ||
|
|
790
|
-
(CONFIG.iiif && CONFIG.iiif.chunkSize) ||
|
|
791
|
-
10
|
|
572
|
+
process.env.CANOPY_CHUNK_SIZE || (cfg.iiif && cfg.iiif.chunkSize) || 20
|
|
792
573
|
)
|
|
793
574
|
);
|
|
794
|
-
const chunks = Math.
|
|
575
|
+
const chunks = Math.ceil(tasks.length / chunkSize);
|
|
576
|
+
// Summary before processing chunks
|
|
795
577
|
try {
|
|
578
|
+
const collectionsCount = visitedCollections.size || 0;
|
|
796
579
|
logLine(
|
|
797
|
-
|
|
798
|
-
"
|
|
580
|
+
`• Fetching ${tasks.length} Manifest(s) in ${chunks} chunk(s) across ${collectionsCount} Collection(s)`,
|
|
581
|
+
"blue",
|
|
582
|
+
{ dim: true }
|
|
799
583
|
);
|
|
800
584
|
} catch (_) {}
|
|
801
|
-
const
|
|
585
|
+
const iiifRecords = [];
|
|
802
586
|
const unsafeThumbs = !!(
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
587
|
+
cfg &&
|
|
588
|
+
cfg.iiif &&
|
|
589
|
+
cfg.iiif.thumbnails &&
|
|
590
|
+
cfg.iiif.thumbnails.unsafe === true
|
|
807
591
|
);
|
|
808
592
|
const thumbSize =
|
|
809
|
-
(
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
593
|
+
(cfg &&
|
|
594
|
+
cfg.iiif &&
|
|
595
|
+
cfg.iiif.thumbnails &&
|
|
596
|
+
cfg.iiif.thumbnails.preferredSize) ||
|
|
813
597
|
1200;
|
|
598
|
+
|
|
599
|
+
// Compile the works layout component once per run
|
|
600
|
+
let WorksLayoutComp = null;
|
|
601
|
+
try {
|
|
602
|
+
const worksLayoutPath = path.join(CONTENT_DIR, "works", "_layout.mdx");
|
|
603
|
+
WorksLayoutComp = await mdx.compileMdxToComponent(worksLayoutPath);
|
|
604
|
+
} catch (_) {
|
|
605
|
+
// Minimal fallback layout if missing or fails to compile
|
|
606
|
+
WorksLayoutComp = function FallbackWorksLayout({ manifest }) {
|
|
607
|
+
const title = firstLabelString(manifest && manifest.label);
|
|
608
|
+
return React.createElement(
|
|
609
|
+
"div",
|
|
610
|
+
{ className: "content" },
|
|
611
|
+
React.createElement("h1", null, title || "Untitled"),
|
|
612
|
+
// Render viewer placeholder for hydration
|
|
613
|
+
React.createElement(
|
|
614
|
+
"div",
|
|
615
|
+
{ "data-canopy-viewer": "1" },
|
|
616
|
+
React.createElement("script", {
|
|
617
|
+
type: "application/json",
|
|
618
|
+
dangerouslySetInnerHTML: {
|
|
619
|
+
__html: JSON.stringify({
|
|
620
|
+
iiifContent: manifest && (manifest.id || ""),
|
|
621
|
+
}),
|
|
622
|
+
},
|
|
623
|
+
})
|
|
624
|
+
)
|
|
625
|
+
);
|
|
626
|
+
};
|
|
627
|
+
}
|
|
628
|
+
|
|
814
629
|
for (let ci = 0; ci < chunks; ci++) {
|
|
815
630
|
const chunk = tasks.slice(ci * chunkSize, (ci + 1) * chunkSize);
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
} catch (_) {}
|
|
631
|
+
logLine(`• Chunk ${ci + 1}/${chunks}`, "blue", { dim: true });
|
|
632
|
+
|
|
819
633
|
const concurrency = Math.max(
|
|
820
634
|
1,
|
|
821
635
|
Number(
|
|
822
636
|
process.env.CANOPY_FETCH_CONCURRENCY ||
|
|
823
|
-
(
|
|
637
|
+
(cfg.iiif && cfg.iiif.concurrency) ||
|
|
824
638
|
6
|
|
825
639
|
)
|
|
826
640
|
);
|
|
@@ -848,18 +662,16 @@ async function buildIiifCollectionPages(CONFIG) {
|
|
|
848
662
|
const idx = next - 1;
|
|
849
663
|
const id = it.id || it["@id"] || "";
|
|
850
664
|
let manifest = await loadCachedManifestById(id);
|
|
851
|
-
// Buffer logs for ordered output
|
|
852
665
|
const lns = [];
|
|
853
|
-
// Logging: cached or fetched
|
|
854
666
|
if (manifest) {
|
|
855
|
-
lns.push([
|
|
667
|
+
lns.push([`↓ ${String(id)} → Cached`, "yellow"]);
|
|
856
668
|
} else if (/^https?:\/\//i.test(String(id || ""))) {
|
|
857
669
|
try {
|
|
858
670
|
const res = await fetch(String(id), {
|
|
859
671
|
headers: { Accept: "application/json" },
|
|
860
672
|
}).catch(() => null);
|
|
861
673
|
if (res && res.ok) {
|
|
862
|
-
lns.push([
|
|
674
|
+
lns.push([`↓ ${String(id)} → ${res.status}`, "yellow"]);
|
|
863
675
|
const remote = await res.json();
|
|
864
676
|
const norm = await normalizeToV3(remote);
|
|
865
677
|
manifest = norm;
|
|
@@ -870,21 +682,20 @@ async function buildIiifCollectionPages(CONFIG) {
|
|
|
870
682
|
);
|
|
871
683
|
} else {
|
|
872
684
|
lns.push([
|
|
873
|
-
|
|
685
|
+
`⊘ ${String(id)} → ${res ? res.status : "ERR"}`,
|
|
874
686
|
"red",
|
|
875
687
|
]);
|
|
876
688
|
continue;
|
|
877
689
|
}
|
|
878
690
|
} catch (e) {
|
|
879
|
-
lns.push([
|
|
691
|
+
lns.push([`⊘ ${String(id)} → ERR`, "red"]);
|
|
880
692
|
continue;
|
|
881
693
|
}
|
|
882
694
|
} else if (/^file:\/\//i.test(String(id || ""))) {
|
|
883
|
-
// Support local manifests via file:// in dev
|
|
884
695
|
try {
|
|
885
696
|
const local = await readJsonFromUri(String(id), { log: false });
|
|
886
697
|
if (!local) {
|
|
887
|
-
lns.push([
|
|
698
|
+
lns.push([`⊘ ${String(id)} → ERR`, "red"]);
|
|
888
699
|
continue;
|
|
889
700
|
}
|
|
890
701
|
const norm = await normalizeToV3(local);
|
|
@@ -894,54 +705,64 @@ async function buildIiifCollectionPages(CONFIG) {
|
|
|
894
705
|
String(id),
|
|
895
706
|
String(it.parent || "")
|
|
896
707
|
);
|
|
897
|
-
lns.push([
|
|
708
|
+
lns.push([`↓ ${String(id)} → Cached`, "yellow"]);
|
|
898
709
|
} catch (_) {
|
|
899
|
-
lns.push([
|
|
710
|
+
lns.push([`⊘ ${String(id)} → ERR`, "red"]);
|
|
900
711
|
continue;
|
|
901
712
|
}
|
|
902
713
|
} else {
|
|
903
|
-
|
|
904
|
-
lns.push([`✗ ${String(id)} ➜ SKIP`, "red"]);
|
|
714
|
+
lns.push([`⊘ ${String(id)} → SKIP`, "red"]);
|
|
905
715
|
continue;
|
|
906
716
|
}
|
|
907
717
|
if (!manifest) continue;
|
|
908
718
|
manifest = await normalizeToV3(manifest);
|
|
909
719
|
const title = firstLabelString(manifest.label);
|
|
910
|
-
const baseSlug =
|
|
720
|
+
const baseSlug =
|
|
721
|
+
slugify(title || "untitled", {
|
|
722
|
+
lower: true,
|
|
723
|
+
strict: true,
|
|
724
|
+
trim: true,
|
|
725
|
+
}) || "untitled";
|
|
911
726
|
const nid = normalizeIiifId(String(manifest.id || id));
|
|
912
|
-
// Use existing mapping if present; otherwise allocate and persist
|
|
913
727
|
let idxMap = await loadManifestIndex();
|
|
914
728
|
idxMap.byId = Array.isArray(idxMap.byId) ? idxMap.byId : [];
|
|
915
|
-
let mEntry = idxMap.byId.find(
|
|
729
|
+
let mEntry = idxMap.byId.find(
|
|
730
|
+
(e) => e && e.type === "Manifest" && normalizeIiifId(e.id) === nid
|
|
731
|
+
);
|
|
916
732
|
let slug = mEntry && mEntry.slug;
|
|
917
733
|
if (!slug) {
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
const
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
734
|
+
slug = computeUniqueSlug(idxMap, baseSlug, nid, "Manifest");
|
|
735
|
+
const parentNorm = normalizeIiifId(String(it.parent || ""));
|
|
736
|
+
const newEntry = {
|
|
737
|
+
id: nid,
|
|
738
|
+
type: "Manifest",
|
|
739
|
+
slug,
|
|
740
|
+
parent: parentNorm,
|
|
741
|
+
};
|
|
742
|
+
const existingIdx = idxMap.byId.findIndex(
|
|
743
|
+
(e) => e && e.type === "Manifest" && normalizeIiifId(e.id) === nid
|
|
744
|
+
);
|
|
745
|
+
if (existingIdx >= 0) idxMap.byId[existingIdx] = newEntry;
|
|
746
|
+
else idxMap.byId.push(newEntry);
|
|
924
747
|
await saveManifestIndex(idxMap);
|
|
925
748
|
}
|
|
926
749
|
const href = path.join("works", slug + ".html");
|
|
927
750
|
const outPath = path.join(OUT_DIR, href);
|
|
928
751
|
ensureDirSync(path.dirname(outPath));
|
|
929
752
|
try {
|
|
930
|
-
// Provide MDX components mapping so tags like <Viewer/> and <HelloWorld/> resolve
|
|
931
753
|
let components = {};
|
|
932
754
|
try {
|
|
933
755
|
components = await import("@canopy-iiif/app/ui");
|
|
934
756
|
} catch (_) {
|
|
935
757
|
components = {};
|
|
936
758
|
}
|
|
937
|
-
const { withBase } = require("
|
|
759
|
+
const { withBase } = require("../common");
|
|
938
760
|
const Anchor = function A(props) {
|
|
939
761
|
let { href = "", ...rest } = props || {};
|
|
940
762
|
href = withBase(href);
|
|
941
763
|
return React.createElement("a", { href, ...rest }, props.children);
|
|
942
764
|
};
|
|
943
765
|
const compMap = { ...components, a: Anchor };
|
|
944
|
-
// Gracefully handle HelloWorld if not provided anywhere
|
|
945
766
|
if (!components.HelloWorld) {
|
|
946
767
|
components.HelloWorld = components.Fallback
|
|
947
768
|
? (props) =>
|
|
@@ -961,7 +782,7 @@ async function buildIiifCollectionPages(CONFIG) {
|
|
|
961
782
|
const { loadAppWrapper } = require("./mdx");
|
|
962
783
|
const app = await loadAppWrapper();
|
|
963
784
|
|
|
964
|
-
const mdxContent = React.createElement(
|
|
785
|
+
const mdxContent = React.createElement(WorksLayoutComp, { manifest });
|
|
965
786
|
const siteTree = app && app.App ? mdxContent : mdxContent;
|
|
966
787
|
const wrappedApp =
|
|
967
788
|
app && app.App
|
|
@@ -988,59 +809,95 @@ async function buildIiifCollectionPages(CONFIG) {
|
|
|
988
809
|
)
|
|
989
810
|
.split(path.sep)
|
|
990
811
|
.join("/");
|
|
991
|
-
// Detect placeholders to decide which runtimes to inject
|
|
992
812
|
const needsHydrateViewer = body.includes("data-canopy-viewer");
|
|
993
813
|
const needsRelated = body.includes("data-canopy-related-items");
|
|
994
|
-
const needsCommand = body.includes(
|
|
995
|
-
const needsHydrate =
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
814
|
+
const needsCommand = body.includes("data-canopy-command");
|
|
815
|
+
const needsHydrate =
|
|
816
|
+
body.includes("data-canopy-hydrate") ||
|
|
817
|
+
needsHydrateViewer ||
|
|
818
|
+
needsRelated ||
|
|
819
|
+
needsCommand;
|
|
999
820
|
|
|
1000
|
-
// Compute script paths relative to the output HTML
|
|
1001
821
|
const viewerRel = needsHydrateViewer
|
|
1002
|
-
? path
|
|
822
|
+
? path
|
|
823
|
+
.relative(
|
|
824
|
+
path.dirname(outPath),
|
|
825
|
+
path.join(OUT_DIR, "scripts", "canopy-viewer.js")
|
|
826
|
+
)
|
|
827
|
+
.split(path.sep)
|
|
828
|
+
.join("/")
|
|
1003
829
|
: null;
|
|
1004
830
|
const sliderRel = needsRelated
|
|
1005
|
-
? path
|
|
831
|
+
? path
|
|
832
|
+
.relative(
|
|
833
|
+
path.dirname(outPath),
|
|
834
|
+
path.join(OUT_DIR, "scripts", "canopy-slider.js")
|
|
835
|
+
)
|
|
836
|
+
.split(path.sep)
|
|
837
|
+
.join("/")
|
|
1006
838
|
: null;
|
|
1007
839
|
const relatedRel = needsRelated
|
|
1008
|
-
? path
|
|
840
|
+
? path
|
|
841
|
+
.relative(
|
|
842
|
+
path.dirname(outPath),
|
|
843
|
+
path.join(OUT_DIR, "scripts", "canopy-related-items.js")
|
|
844
|
+
)
|
|
845
|
+
.split(path.sep)
|
|
846
|
+
.join("/")
|
|
1009
847
|
: null;
|
|
1010
848
|
const commandRel = needsCommand
|
|
1011
|
-
? path
|
|
849
|
+
? path
|
|
850
|
+
.relative(
|
|
851
|
+
path.dirname(outPath),
|
|
852
|
+
path.join(OUT_DIR, "scripts", "canopy-command.js")
|
|
853
|
+
)
|
|
854
|
+
.split(path.sep)
|
|
855
|
+
.join("/")
|
|
1012
856
|
: null;
|
|
1013
857
|
|
|
1014
|
-
// Choose a main script so execution order is preserved (builder before slider)
|
|
1015
858
|
let jsRel = null;
|
|
1016
859
|
if (needsRelated && sliderRel) jsRel = sliderRel;
|
|
1017
860
|
else if (viewerRel) jsRel = viewerRel;
|
|
1018
861
|
|
|
1019
|
-
// Include hydration scripts via htmlShell
|
|
1020
862
|
let headExtra = head;
|
|
1021
|
-
// Ensure React globals are present only when client React is needed
|
|
1022
863
|
const needsReact = !!(needsHydrateViewer || needsRelated);
|
|
1023
|
-
let vendorTag =
|
|
864
|
+
let vendorTag = "";
|
|
1024
865
|
if (needsReact) {
|
|
1025
866
|
try {
|
|
1026
|
-
const vendorAbs = path.join(
|
|
1027
|
-
|
|
1028
|
-
|
|
867
|
+
const vendorAbs = path.join(
|
|
868
|
+
OUT_DIR,
|
|
869
|
+
"scripts",
|
|
870
|
+
"react-globals.js"
|
|
871
|
+
);
|
|
872
|
+
let vendorRel = path
|
|
873
|
+
.relative(path.dirname(outPath), vendorAbs)
|
|
874
|
+
.split(path.sep)
|
|
875
|
+
.join("/");
|
|
876
|
+
try {
|
|
877
|
+
const stv = fs.statSync(vendorAbs);
|
|
878
|
+
vendorRel += `?v=${Math.floor(stv.mtimeMs || Date.now())}`;
|
|
879
|
+
} catch (_) {}
|
|
1029
880
|
vendorTag = `<script src="${vendorRel}"></script>`;
|
|
1030
881
|
} catch (_) {}
|
|
1031
882
|
}
|
|
1032
|
-
// Prepend additional scripts so the selected jsRel runs last
|
|
1033
883
|
const extraScripts = [];
|
|
1034
|
-
if (relatedRel && jsRel !== relatedRel)
|
|
1035
|
-
|
|
1036
|
-
if (
|
|
1037
|
-
|
|
1038
|
-
if (
|
|
1039
|
-
|
|
1040
|
-
|
|
884
|
+
if (relatedRel && jsRel !== relatedRel)
|
|
885
|
+
extraScripts.push(`<script defer src="${relatedRel}"></script>`);
|
|
886
|
+
if (viewerRel && jsRel !== viewerRel)
|
|
887
|
+
extraScripts.push(`<script defer src="${viewerRel}"></script>`);
|
|
888
|
+
if (sliderRel && jsRel !== sliderRel)
|
|
889
|
+
extraScripts.push(`<script defer src="${sliderRel}"></script>`);
|
|
890
|
+
if (commandRel && jsRel !== commandRel)
|
|
891
|
+
extraScripts.push(`<script defer src="${commandRel}"></script>`);
|
|
892
|
+
if (extraScripts.length)
|
|
893
|
+
headExtra = extraScripts.join("") + headExtra;
|
|
1041
894
|
try {
|
|
1042
|
-
const { BASE_PATH } = require(
|
|
1043
|
-
if (BASE_PATH)
|
|
895
|
+
const { BASE_PATH } = require("../common");
|
|
896
|
+
if (BASE_PATH)
|
|
897
|
+
vendorTag =
|
|
898
|
+
`<script>window.CANOPY_BASE_PATH=${JSON.stringify(
|
|
899
|
+
BASE_PATH
|
|
900
|
+
)}</script>` + vendorTag;
|
|
1044
901
|
} catch (_) {}
|
|
1045
902
|
let pageBody = body;
|
|
1046
903
|
let html = htmlShell({
|
|
@@ -1051,24 +908,23 @@ async function buildIiifCollectionPages(CONFIG) {
|
|
|
1051
908
|
headExtra: vendorTag + headExtra,
|
|
1052
909
|
});
|
|
1053
910
|
try {
|
|
1054
|
-
html = require("
|
|
911
|
+
html = require("../common").applyBaseToHtml(html);
|
|
1055
912
|
} catch (_) {}
|
|
1056
913
|
await fsp.writeFile(outPath, html, "utf8");
|
|
1057
914
|
lns.push([
|
|
1058
|
-
|
|
915
|
+
`✔ Created ${path.relative(process.cwd(), outPath)}`,
|
|
1059
916
|
"green",
|
|
1060
917
|
]);
|
|
1061
|
-
// Resolve thumbnail URL and dimensions for this manifest (safe by default; expanded "unsafe" if configured)
|
|
1062
918
|
let thumbUrl = "";
|
|
1063
919
|
let thumbWidth = undefined;
|
|
1064
920
|
let thumbHeight = undefined;
|
|
1065
921
|
try {
|
|
1066
|
-
const { getThumbnail } = require("
|
|
922
|
+
const { getThumbnail } = require("../iiif/thumbnail");
|
|
1067
923
|
const t = await getThumbnail(manifest, thumbSize, unsafeThumbs);
|
|
1068
924
|
if (t && t.url) {
|
|
1069
925
|
thumbUrl = String(t.url);
|
|
1070
|
-
thumbWidth = typeof t.width ===
|
|
1071
|
-
thumbHeight = typeof t.height ===
|
|
926
|
+
thumbWidth = typeof t.width === "number" ? t.width : undefined;
|
|
927
|
+
thumbHeight = typeof t.height === "number" ? t.height : undefined;
|
|
1072
928
|
const idx = await loadManifestIndex();
|
|
1073
929
|
if (Array.isArray(idx.byId)) {
|
|
1074
930
|
const entry = idx.byId.find(
|
|
@@ -1079,23 +935,25 @@ async function buildIiifCollectionPages(CONFIG) {
|
|
|
1079
935
|
);
|
|
1080
936
|
if (entry) {
|
|
1081
937
|
entry.thumbnail = String(thumbUrl);
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
if (typeof thumbHeight ===
|
|
938
|
+
if (typeof thumbWidth === "number")
|
|
939
|
+
entry.thumbnailWidth = thumbWidth;
|
|
940
|
+
if (typeof thumbHeight === "number")
|
|
941
|
+
entry.thumbnailHeight = thumbHeight;
|
|
1085
942
|
await saveManifestIndex(idx);
|
|
1086
943
|
}
|
|
1087
944
|
}
|
|
1088
945
|
}
|
|
1089
946
|
} catch (_) {}
|
|
1090
|
-
|
|
1091
|
-
searchRecords.push({
|
|
947
|
+
iiifRecords.push({
|
|
1092
948
|
id: String(manifest.id || id),
|
|
1093
949
|
title,
|
|
1094
950
|
href: href.split(path.sep).join("/"),
|
|
1095
951
|
type: "work",
|
|
1096
952
|
thumbnail: thumbUrl || undefined,
|
|
1097
|
-
thumbnailWidth:
|
|
1098
|
-
|
|
953
|
+
thumbnailWidth:
|
|
954
|
+
typeof thumbWidth === "number" ? thumbWidth : undefined,
|
|
955
|
+
thumbnailHeight:
|
|
956
|
+
typeof thumbHeight === "number" ? thumbHeight : undefined,
|
|
1099
957
|
});
|
|
1100
958
|
} catch (e) {
|
|
1101
959
|
lns.push([
|
|
@@ -1113,16 +971,16 @@ async function buildIiifCollectionPages(CONFIG) {
|
|
|
1113
971
|
);
|
|
1114
972
|
await Promise.all(workers);
|
|
1115
973
|
}
|
|
1116
|
-
return {
|
|
974
|
+
return { iiifRecords };
|
|
1117
975
|
}
|
|
1118
976
|
|
|
1119
977
|
module.exports = {
|
|
1120
978
|
buildIiifCollectionPages,
|
|
1121
979
|
loadConfig,
|
|
1122
|
-
// Expose for other build steps that need to annotate cache metadata
|
|
1123
980
|
loadManifestIndex,
|
|
1124
981
|
saveManifestIndex,
|
|
1125
982
|
};
|
|
983
|
+
|
|
1126
984
|
// Debug: list collections cache after traversal
|
|
1127
985
|
try {
|
|
1128
986
|
if (process.env.CANOPY_IIIF_DEBUG === "1") {
|