@rr0/cms 0.3.58 → 0.3.60

package/dist/CMSGenerator.js

@@ -103,7 +103,15 @@ export class CMSGenerator {
         const peopleRenderer = new PeopleHtmlRenderer();
         const { peopleService, peopleSteps } = await this.setupPeople(context, peopleRenderer, this.options.copies);
         const timeTextBuilder = this.timeTextBuilder;
-        const searchVisitor = new SearchVisitor({ notIndexedUrls: ["404.html", "Referencement.html"], indexWords: false }, timeTextBuilder);
+        const searchVisitor = new SearchVisitor({
+            notIndexedUrls: [
+                "404.html",
+                "Referencement.html",
+                "org/ca/company/avro/Avrocar/index.html",
+                "org/renseignement.html",
+                "time/1/9/7/7/Poher_Matrice/app/dist/index.html"
+            ], indexWords: false
+        }, timeTextBuilder);
         const { sourceRenderer, sourceFactory, sourceReplacerFactory } = this.setupSources(timeTextBuilder, timeFormat);
         const { noteRenderer, noteReplacerFactory } = this.setupNotes();
         const caseRenderer = new CaseSummaryRenderer(noteRenderer, sourceFactory, sourceRenderer, timeElementFactory);
@@ -188,7 +196,7 @@ export class CMSGenerator {
         }
         const reindex = args.reindex;
         if (reindex === null || reindex === void 0 ? void 0 : reindex.includes("search")) {
-            ssg.add(new SearchIndexStep("search/index.json", searchVisitor));
+            ssg.add(new SearchIndexStep("search/index.json", searchVisitor, outDir, (context, fileName) => timeService.setContextFromFile(context, fileName)));
         }
         if (reindex === null || reindex === void 0 ? void 0 : reindex.includes("sources")) {
             ssg.add(new SourceIndexStep(this.options.sourceRegistryFileName, sourceFactory));
@@ -212,6 +220,7 @@ export class CMSGenerator {
             catch (e) {
                 context.error(err);
             }
+            throw err;
         }
         finally {
             console.timeEnd("ssg");

package/dist/DataContentVisitor.test.js

@@ -18,7 +18,7 @@ describe("DataContentVisitor", () => {
     };
     test("insert portrait image when contents has no image", async () => {
         var _a, _b;
-        const context = cmsTestUtil.newHtmlContext("people/v/VertongenJeanLuc/index.html", '<div class="contents"><p>Biographie</p></div>');
+        const context = cmsTestUtil.newHtmlContext("people/v/VertongenJeanLuc/index.html", "<div class=\"contents\"><p>Biographie</p></div>");
         const visitor = new TestDataContentVisitor();
         await visitor.renderImage(context, portraitEvent);
         const figure = context.file.document.querySelector(".contents > figure");
@@ -26,9 +26,9 @@ describe("DataContentVisitor", () => {
         expect((_b = figure === null || figure === void 0 ? void 0 : figure.querySelector("figcaption")) === null || _b === void 0 ? void 0 : _b.textContent).toBe("Portrait");
     });
     test("does not insert a portrait image already present", async () => {
-        const context = cmsTestUtil.newHtmlContext("people/v/VertongenJeanLuc/index.html", '<div class="contents"><img src="portrait.jpg"><p>Biographie</p></div>');
+        const context = cmsTestUtil.newHtmlContext("people/v/VertongenJeanLuc/index.html", "<div class=\"contents\"><img src=\"portrait.jpg\"><p>Biographie</p></div>");
         const visitor = new TestDataContentVisitor();
         await visitor.renderImage(context, portraitEvent);
-        expect(context.file.document.querySelectorAll('.contents img[src="portrait.jpg"]').length).toBe(1);
+        expect(context.file.document.querySelectorAll(".contents img[src=\"portrait.jpg\"]").length).toBe(1);
     });
 });

package/dist/search/SearchIndexStep.d.ts

@@ -1,21 +1,26 @@
 import { SsgContext, SsgStep } from "ssg-api";
 import { SearchVisitor } from "./SearchVisitor.js";
+import { HtmlRR0Context } from "../RR0Context.js";
 /**
  * Saves the index file collected by the SearchCommand.
  */
 export declare class SearchIndexStep implements SsgStep {
     protected fileName: string;
     protected searchCommand: SearchVisitor;
+    protected contentRoot?: string;
+    protected prepareContext?: (context: HtmlRR0Context, fileName: string) => void;
     protected encoding: BufferEncoding;
     /**
      * @param fileName The index file path
      * @param searchCommand The command that collected the pages info.
      */
-    constructor(fileName: string, searchCommand: SearchVisitor);
+    constructor(fileName: string, searchCommand: SearchVisitor, contentRoot?: string, prepareContext?: (context: HtmlRR0Context, fileName: string) => void);
     /**
      * Write the search index file.
      *
      * @param context
      */
     execute(context: SsgContext): Promise<any>;
+    protected decodeTitle(title: string): string;
+    protected readSourceTitle(sourceFile: string): string;
 }

package/dist/search/SearchIndexStep.js

@@ -1,5 +1,8 @@
+import { SearchVisitor } from "./SearchVisitor.js";
 import fs from "fs";
 import { writeFile } from "@javarome/fileutil";
+import path from "path";
+import { glob } from "glob";
 /**
  * Saves the index file collected by the SearchCommand.
  */
@@ -8,9 +11,11 @@ export class SearchIndexStep {
      * @param fileName The index file path
      * @param searchCommand The command that collected the pages info.
      */
-    constructor(fileName, searchCommand) {
+    constructor(fileName, searchCommand, contentRoot, prepareContext) {
         this.fileName = fileName;
         this.searchCommand = searchCommand;
+        this.contentRoot = contentRoot;
+        this.prepareContext = prepareContext;
         this.encoding = "utf8";
     }
     /**
@@ -18,35 +23,104 @@ export class SearchIndexStep {
      *
      * @param context
      */
-    execute(context) {
+    async execute(context) {
+        var _a;
         const newIndex = this.searchCommand.index;
         let existingIndex;
-        try {
-            existingIndex = JSON.parse(fs.readFileSync(this.fileName, { encoding: this.encoding }));
-            const newPages = newIndex.pages;
-            const existingPages = existingIndex.pages;
-            for (let newPage of newPages) {
-                const alreadyIndexedPage = existingPages.find(existingPage => existingPage.url === newPage.url);
-                if (alreadyIndexedPage) {
-                    Object.assign(alreadyIndexedPage, newPage);
+        if (this.contentRoot) {
+            const pagesByUrl = new Map();
+            const pagesByResultTitle = new Map();
+            const duplicateTitles = [];
+            const outputFiles = (await glob(path.join(this.contentRoot, "**/*.html"))).sort();
+            const sourceRoot = path.dirname(path.dirname(this.fileName));
+            for (const outputFile of outputFiles) {
+                const url = path.relative(this.contentRoot, outputFile);
+                const sourceFile = path.resolve(sourceRoot, url);
+                if (!fs.existsSync(sourceFile)) {
+                    continue;
                 }
-                else {
-                    existingPages.push(newPage);
+                const pageContext = context.clone();
+                (_a = this.prepareContext) === null || _a === void 0 ? void 0 : _a.call(this, pageContext, outputFile);
+                const html = fs.readFileSync(outputFile, this.encoding);
+                const titleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
+                const outputTitle = titleMatch ? this.decodeTitle(titleMatch[1]) : "";
+                const title = this.readSourceTitle(sourceFile) || outputTitle;
+                const pageInfo = this.searchCommand.pageInfoFrom(title, url.split(path.sep).join("/"), pageContext);
+                if (pageInfo) {
+                    const resultTitle = SearchVisitor.resultTitle(pageInfo);
+                    const titleIndexed = pagesByResultTitle.get(resultTitle);
+                    if (titleIndexed && titleIndexed.url !== pageInfo.url) {
+                        duplicateTitles.push(`Search result "${resultTitle}" with URL ${pageInfo.url} is already indexed with URL ${titleIndexed.url}`);
+                        continue;
+                    }
+                    pagesByUrl.set(pageInfo.url, pageInfo);
+                    pagesByResultTitle.set(resultTitle, pageInfo);
                 }
             }
-            existingPages.sort((pageInfo1, pageInfo2) => pageInfo1.title > pageInfo2.title ? 1 : pageInfo1.title < pageInfo2.title ? -1 : 0);
-        }
-        catch (e) {
-            if (e.errno !== -2) {
-                throw e;
+            if (duplicateTitles.length > 0) {
+                throw new Error(`${duplicateTitles.length} indistinguishable search results:\n${duplicateTitles.join("\n")}`);
             }
-            context.warn("Could not find", this.fileName, "Will create it");
-            existingIndex = newIndex;
+            const pages = Array.from(pagesByUrl.values());
+            pages.sort((pageInfo1, pageInfo2) => pageInfo1.title.localeCompare(pageInfo2.title));
+            existingIndex = { pages, words: newIndex.words };
         }
+        else
+            try {
+                existingIndex = JSON.parse(fs.readFileSync(this.fileName, { encoding: this.encoding }));
+                const newPages = newIndex.pages;
+                const contentRoot = path.dirname(path.dirname(this.fileName));
+                const pagesByUrl = new Map(existingIndex.pages
+                    .filter(page => fs.existsSync(path.resolve(contentRoot, page.url)))
+                    .map(page => [page.url, page]));
+                for (const newPage of newPages) {
+                    pagesByUrl.set(newPage.url, newPage);
+                }
+                existingIndex.pages = Array.from(pagesByUrl.values());
+                existingIndex.pages.sort((pageInfo1, pageInfo2) => pageInfo1.title > pageInfo2.title ? 1 : pageInfo1.title < pageInfo2.title ? -1 : 0);
+            }
+            catch (e) {
+                if (e.errno !== -2) {
+                    throw e;
+                }
+                context.warn("Could not find", this.fileName, "Will create it");
+                existingIndex = newIndex;
+            }
         const indexSize = existingIndex.pages.length;
         context.setVar("indexSize", indexSize);
         context.log("Saving search index of", indexSize, "pages at", this.fileName);
         const indexJson = JSON.stringify(existingIndex);
         return writeFile(this.fileName, indexJson, "utf-8");
     }
+    decodeTitle(title) {
+        return title
+            .replace(/<[^>]+>/g, "")
+            .replace(/&#(\d+);/g, (_match, code) => String.fromCodePoint(Number(code)))
+            .replace(/&#x([\da-f]+);/gi, (_match, code) => String.fromCodePoint(Number.parseInt(code, 16)))
+            .replace(/&nbsp;/gi, " ")
+            .replace(/&amp;/gi, "&")
+            .replace(/&quot;/gi, "\"")
+            .replace(/&apos;|&#39;/gi, "'")
+            .replace(/&lt;/gi, "<")
+            .replace(/&gt;/gi, ">")
+            .replace(/\s+/g, " ")
+            .trim();
+    }
+    readSourceTitle(sourceFile) {
+        const source = fs.readFileSync(sourceFile, this.encoding);
+        const htmlTitleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(source);
+        if (htmlTitleMatch) {
+            return this.decodeTitle(htmlTitleMatch[1]);
+        }
+        for (const directiveMatch of source.matchAll(/<!--#set\s+([\s\S]*?)-->/gi)) {
+            const attributes = directiveMatch[1];
+            if (!/\bvar\s*=\s*(["'])title\1/i.test(attributes)) {
+                continue;
+            }
+            const valueMatch = /\bvalue\s*=\s*(["'])([\s\S]*?)\1/i.exec(attributes);
+            if (valueMatch) {
+                return this.decodeTitle(valueMatch[2]);
+            }
+        }
+        return "";
+    }
 }

package/dist/search/SearchIndexStep.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/search/SearchIndexStep.test.js

@@ -0,0 +1,177 @@
+import { describe, expect, test } from "@javarome/testscript";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { SsgContextImpl } from "ssg-api";
+import { SearchIndexStep } from "./SearchIndexStep.js";
+describe("SearchIndexStep", () => {
+    test("removes missing and duplicate pages while updating the current pages", async () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), "rr0-search-index-"));
+        try {
+            const searchDir = path.join(root, "search");
+            fs.mkdirSync(searchDir);
+            fs.writeFileSync(path.join(root, "current.html"), "");
+            const fileName = path.join(searchDir, "index.json");
+            const existingIndex = {
+                pages: [
+                    { title: "Old current title", url: "current.html", time: "" },
+                    { title: "Duplicate current title", url: "current.html", time: "" },
+                    { title: "Moved page", url: "old/path.html", time: "" }
+                ],
+                words: {}
+            };
+            fs.writeFileSync(fileName, JSON.stringify(existingIndex));
+            const newIndex = {
+                pages: [
+                    { title: "Current title", url: "current.html", time: "" },
+                    { title: "New page", url: "new.html", time: "" }
+                ],
+                words: {}
+            };
+            const visitor = { index: newIndex };
+            const context = {
+                setVar() {
+                },
+                log() {
+                },
+                warn() {
+                }
+            };
+            await new SearchIndexStep(fileName, visitor).execute(context);
+            const savedIndex = JSON.parse(fs.readFileSync(fileName, "utf8"));
+            expect(savedIndex.pages).toEqual([
+                { title: "Current title", url: "current.html", time: "" },
+                { title: "New page", url: "new.html", time: "" }
+            ]);
+        }
+        finally {
+            fs.rmSync(root, { recursive: true, force: true });
+        }
+    });
+    test("rebuilds the index from every HTML page in the output directory", async () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), "rr0-search-index-"));
+        try {
+            const outDir = path.join(root, "out");
+            const searchDir = path.join(root, "search");
+            fs.mkdirSync(path.join(outDir, "science"), { recursive: true });
+            fs.mkdirSync(searchDir);
+            fs.mkdirSync(path.join(root, "science"));
+            fs.writeFileSync(path.join(outDir, "science", "UfoAtHome.html"), "<html><head><title>UFO@home</title></head><body></body></html>");
+            fs.writeFileSync(path.join(outDir, "other.html"), "<html><head><title>Other page</title></head><body></body></html>");
+            fs.writeFileSync(path.join(root, "science", "UfoAtHome.html"), "");
+            fs.writeFileSync(path.join(root, "other.html"), "");
+            fs.writeFileSync(path.join(outDir, "stale.html"), "<html><head><title>Stale page</title></head><body></body></html>");
+            const fileName = path.join(searchDir, "index.json");
+            const newIndex = { pages: [], words: {} };
+            const visitor = {
+                index: newIndex,
+                pageInfoFrom(title, url) {
+                    return {
+                        title,
+                        url,
+                        time: ""
+                    };
+                }
+            };
+            const context = new SsgContextImpl("fr");
+            await new SearchIndexStep(fileName, visitor, outDir).execute(context);
+            const savedIndex = JSON.parse(fs.readFileSync(fileName, "utf8"));
+            expect(savedIndex.pages).toEqual([
+                { title: "Other page", url: "other.html", time: "" },
+                { title: "UFO@home", url: "science/UfoAtHome.html", time: "" }
+            ]);
+        }
+        finally {
+            fs.rmSync(root, { recursive: true, force: true });
+        }
+    });
+    test("rejects the same title at two different URLs", async () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), "rr0-search-index-"));
+        try {
+            const outDir = path.join(root, "out");
+            const searchDir = path.join(root, "search");
+            fs.mkdirSync(outDir);
+            fs.mkdirSync(searchDir);
+            fs.writeFileSync(path.join(outDir, "first.html"), "<html><head><title>Same title</title></head><body></body></html>");
+            fs.writeFileSync(path.join(outDir, "second.html"), "<html><head><title>Same title</title></head><body></body></html>");
+            fs.writeFileSync(path.join(root, "first.html"), "");
+            fs.writeFileSync(path.join(root, "second.html"), "");
+            const visitor = {
+                index: { pages: [], words: {} },
+                pageInfoFrom(title, url) {
+                    return {
+                        title,
+                        url,
+                        time: ""
+                    };
+                }
+            };
+            const context = new SsgContextImpl("fr");
+            let error;
+            try {
+                await new SearchIndexStep(path.join(searchDir, "index.json"), visitor, outDir).execute(context);
+            }
+            catch (e) {
+                error = e;
+            }
+            expect(error === null || error === void 0 ? void 0 : error.message.includes("Search result \"Same title\"")).toEqual(true);
+        }
+        finally {
+            fs.rmSync(root, { recursive: true, force: true });
+        }
+    });
+    test("allows identical document titles when their search result dates differ", async () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), "rr0-search-index-"));
+        try {
+            const outDir = path.join(root, "out");
+            const searchDir = path.join(root, "search");
+            fs.mkdirSync(outDir);
+            fs.mkdirSync(searchDir);
+            for (const fileName of ["first.html", "second.html"]) {
+                fs.writeFileSync(path.join(outDir, fileName), "<html><head><title>Same title</title></head><body></body></html>");
+                fs.writeFileSync(path.join(root, fileName), "");
+            }
+            const visitor = {
+                index: { pages: [], words: {} },
+                pageInfoFrom(title, url) {
+                    return { title, url, time: url === "first.html" ? "1900" : "1901" };
+                }
+            };
+            const context = new SsgContextImpl("fr");
+            const fileName = path.join(searchDir, "index.json");
+            await new SearchIndexStep(fileName, visitor, outDir).execute(context);
+            const savedIndex = JSON.parse(fs.readFileSync(fileName, "utf8"));
+            expect(savedIndex.pages.length).toEqual(2);
+        }
+        finally {
+            fs.rmSync(root, { recursive: true, force: true });
+        }
+    });
+    test("prefers an explicit source title over a generated chronological title", async () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), "rr0-search-index-"));
+        try {
+            const outDir = path.join(root, "out");
+            const searchDir = path.join(root, "search");
+            fs.mkdirSync(outDir);
+            fs.mkdirSync(searchDir);
+            fs.writeFileSync(path.join(outDir, "article.html"), "<html><head><title>Mai 1954</title></head><body></body></html>");
+            fs.writeFileSync(path.join(root, "article.html"), "<!--#set var=\"title\" value=\"Canada Hunts for Saucers\" --><!--#include virtual=\"/header.html\" -->");
+            const visitor = {
+                index: { pages: [], words: {} },
+                pageInfoFrom(title, url) {
+                    return { title, url, time: "mai 1954" };
+                }
+            };
+            const context = new SsgContextImpl("fr");
+            const fileName = path.join(searchDir, "index.json");
+            await new SearchIndexStep(fileName, visitor, outDir).execute(context);
+            const savedIndex = JSON.parse(fs.readFileSync(fileName, "utf8"));
+            expect(savedIndex.pages).toEqual([
+                { title: "Canada Hunts for Saucers", url: "article.html", time: "mai 1954" }
+            ]);
+        }
+        finally {
+            fs.rmSync(root, { recursive: true, force: true });
+        }
+    });
+});

package/dist/search/SearchVisitor.d.ts

@@ -30,12 +30,15 @@ export type SearchCommandConfig = {
 export declare class SearchVisitor implements FileVisitor {
     protected config: SearchCommandConfig;
     protected timeTextBuilder: TimeTextBuilder;
+    static resultTitle(pageInfo: PageInfo): string;
     readonly index: SearchIndex;
     protected readonly contentStream: fs.WriteStream | undefined;
     constructor(config: SearchCommandConfig, timeTextBuilder: TimeTextBuilder);
     contentStepEnd(): Promise<void>;
     visit(context: HtmlRR0Context): Promise<void>;
-    protected handleAlreadyIndexed(title: string, url: string, titleIndexed: PageInfo): void;
+    pageInfo(context: HtmlRR0Context, contentRoot?: string): PageInfo | undefined;
+    pageInfoFrom(title: string, url: string, context: HtmlRR0Context): PageInfo | undefined;
+    protected handleAlreadyIndexed(resultTitle: string, url: string, titleIndexed: PageInfo): void;
     protected getContents(doc: Document): string;
     protected indexContent(context: HtmlRR0Context, outputFile: HtmlFileContents): void;
     protected indexWords(context: HtmlRR0Context, outputFile: HtmlFileContents): void;

package/dist/search/SearchVisitor.js

@@ -1,8 +1,13 @@
 import fs from "fs";
+import path from "path";
 /**
  * Builds an index of pages.
  */
 export class SearchVisitor {
+    static resultTitle(pageInfo) {
+        const { title, time } = pageInfo;
+        return title + (time && time !== title.toLowerCase() ? ` (${time})` : "");
+    }
     constructor(config, timeTextBuilder) {
         this.config = config;
         this.timeTextBuilder = timeTextBuilder;
@@ -24,17 +29,21 @@ export class SearchVisitor {
     }
     async visit(context) {
         const file = context.file;
-        const title = file.title;
-        const outDir = "out/";
-        const url = file.name.startsWith(outDir) ? file.name.substring(outDir.length) : file.name;
-        if (title && !this.config.notIndexedUrls.includes(url)) {
+        const pageInfo = this.pageInfo(context);
+        if (pageInfo) {
             const indexedPages = this.index.pages;
-            const titleIndexed = indexedPages.find(page => page.title === title && page.url !== url);
+            const pageIndex = indexedPages.findIndex(page => page.url === pageInfo.url);
+            const resultTitle = SearchVisitor.resultTitle(pageInfo);
+            const titleIndexed = indexedPages.find(page => SearchVisitor.resultTitle(page) === resultTitle && page.url !== pageInfo.url);
             if (titleIndexed) {
-                this.handleAlreadyIndexed(title, url, titleIndexed);
+                this.handleAlreadyIndexed(resultTitle, pageInfo.url, titleIndexed);
+            }
+            if (pageIndex >= 0) {
+                indexedPages[pageIndex] = pageInfo;
+            }
+            else {
+                indexedPages.push(pageInfo);
             }
-            const time = this.timeTextBuilder.build(context, { year: "numeric", month: "short", day: "numeric" }).toLowerCase();
-            indexedPages.push({ title, url, time });
         }
         if (this.config.indexWords) {
             this.indexWords(context, file);
@@ -43,8 +52,22 @@ export class SearchVisitor {
             this.indexContent(context, file);
         }
     }
-    handleAlreadyIndexed(title, url, titleIndexed) {
-        throw new Error(`Title "${title}" with URL ${url} is already indexed with URL ${titleIndexed.url}`);
+    pageInfo(context, contentRoot) {
+        const file = context.file;
+        const url = contentRoot
+            ? path.relative(contentRoot, file.name).split(path.sep).join("/")
+            : file.name.startsWith("out/") ? file.name.substring("out/".length) : file.name;
+        return this.pageInfoFrom(file.title, url, context);
+    }
+    pageInfoFrom(title, url, context) {
+        if (!title || this.config.notIndexedUrls.includes(url)) {
+            return undefined;
+        }
+        const time = this.timeTextBuilder.build(context, { year: "numeric", month: "short", day: "numeric" }).toLowerCase();
+        return { title, url, time };
+    }
+    handleAlreadyIndexed(resultTitle, url, titleIndexed) {
+        throw new Error(`Search result "${resultTitle}" with URL ${url} is already indexed with URL ${titleIndexed.url}`);
     }
     getContents(doc) {
         const div = doc.createElement("div");

package/dist/time/datasource/CsvMapper.js

@@ -64,7 +64,8 @@ export class CsvMapper {
      */
     mapAll(context, sourceCases, sourceTime) {
         const values = sourceCases.map(c => this.map(context, c, sourceTime));
-        return Array.from(this.fields).join(this.sep) + "\n" + values.join("\n");
+        const header = Array.from(this.fields).sort((field1, field2) => field1.localeCompare(field2));
+        return header.join(this.sep) + "\n" + values.join("\n");
     }
     escape(value, force) {
         if (this.escapeStr && (force || value.indexOf(this.sep) >= 0)) {

package/package.json

@@ -2,7 +2,7 @@
   "name": "@rr0/cms",
   "type": "module",
   "author": "Jérôme Beau <rr0@rr0.org> (https://rr0.org)",
-  "version": "0.3.58",
+  "version": "0.3.60",
   "description": "RR0 Content Management System (CMS)",
   "exports": "./dist/index.js",
   "types": "./dist/index.d.ts",