relaton 3.0.7 → 3.1.0

data/lib/relaton/cli/frontend/src/lib/shards.test.ts

@@ -0,0 +1,319 @@
+// @vitest-environment happy-dom
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  createDetailLoader,
+  loadSearchShards,
+  shardPath,
+} from "./shards";
+import type { CompactRecord, DetailRecord, ShardInfo } from "./types";
+import { deferred, stubFetch } from "../test/fetch";
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+function info(over: Partial<ShardInfo> = {}): ShardInfo {
+  return {
+    total: 0,
+    shards: 0,
+    shardSize: 3,
+    detailShards: 0,
+    detailShardSize: 2,
+    ...over,
+  };
+}
+
+function rows(...ids: string[]): CompactRecord[] {
+  return ids.map((id) => ({ r: id, c: `Title ${id}` }));
+}
+
+describe("shardPath", () => {
+  it("zero-pads to four digits, matching the generator", () => {
+    expect(shardPath("search", 0)).toBe("search-0000.json");
+    expect(shardPath("search", 7)).toBe("search-0007.json");
+    expect(shardPath("detail", 1234)).toBe("detail-1234.json");
+  });
+});
+
+describe("loadSearchShards", () => {
+  it("does nothing when there are no shards", async () => {
+    const onBatch = vi.fn();
+    await loadSearchShards(info({ shards: 0 }), onBatch);
+    expect(onBatch).not.toHaveBeenCalled();
+  });
+
+  it("maps compact records onto documents", async () => {
+    stubFetch({
+      "search-0000.json": [
+        { r: "A", c: "Title A", t: "standard", s: "published", d: "2020-01-01", u: "a.yaml", l: "https://x/a" },
+      ],
+    });
+    const batches: string[][] = [];
+    await loadSearchShards(info({ shards: 1 }), (docs) =>
+      batches.push(docs.map((d) => d.id)),
+    );
+    expect(batches[batches.length - 1]).toEqual(["A"]);
+  });
+
+  // The detail shards are addressed by corpus position, so a shard that arrives
+  // early must not jump the queue.
+  it("appends in shard order even when responses resolve out of order", async () => {
+    const first = deferred<CompactRecord[]>();
+    stubFetch({
+      "search-0000.json": () => first.promise,
+      "search-0001.json": rows("B1", "B2"),
+      "search-0002.json": rows("C1"),
+    });
+
+    const seen: string[][] = [];
+    const done = loadSearchShards(
+      info({ shards: 3 }),
+      (docs) => seen.push(docs.map((d) => d.id)),
+      { flushMs: 0 },
+    );
+
+    // Let shards 1 and 2 settle while 0 is still outstanding.
+    await Promise.resolve();
+    await Promise.resolve();
+    first.resolve(rows("A1", "A2"));
+    await done;
+
+    expect(seen[seen.length - 1]).toEqual(["A1", "A2", "B1", "B2", "C1"]);
+    // Nothing was emitted before shard 0 landed.
+    expect(seen.every((batch) => batch.length === 0 || batch[0] === "A1")).toBe(true);
+  });
+
+  it("coalesces emissions instead of firing once per shard", async () => {
+    stubFetch({
+      "search-0000.json": rows("A"),
+      "search-0001.json": rows("B"),
+      "search-0002.json": rows("C"),
+      "search-0003.json": rows("D"),
+    });
+
+    const onBatch = vi.fn();
+    // A clock that never advances: the first batch still paints immediately, the
+    // two in the middle are suppressed, and the final forced emit carries them.
+    await loadSearchShards(info({ shards: 4 }), onBatch, {
+      flushMs: 10_000,
+      now: () => 1_000,
+      concurrency: 1,
+    });
+
+    expect(onBatch).toHaveBeenCalledTimes(2);
+    const ids = (call: number) =>
+      onBatch.mock.calls[call][0].map((d: { id: string }) => d.id);
+    expect(ids(0)).toEqual(["A"]);
+    expect(ids(1)).toEqual(["A", "B", "C", "D"]);
+  });
+
+  it("paints the first batch immediately regardless of the clock's epoch", async () => {
+    stubFetch({
+      "search-0000.json": rows("A"),
+      "search-0001.json": rows("B"),
+    });
+
+    const onBatch = vi.fn();
+    // A monotonic-style clock starting near zero would, with a naive lastEmit of
+    // 0, suppress the very first paint.
+    await loadSearchShards(info({ shards: 2 }), onBatch, {
+      flushMs: 750,
+      now: () => 5,
+      concurrency: 1,
+    });
+
+    expect(onBatch.mock.calls[0][0].map((d: { id: string }) => d.id)).toEqual(["A"]);
+  });
+
+  it("stamps each document with its corpus position", async () => {
+    stubFetch({
+      "search-0000.json": rows("A1", "A2", "A3"),
+      "search-0001.json": rows("B1", "B2"),
+    });
+
+    let docs: { id: string; pos?: number }[] = [];
+    await loadSearchShards(info({ shards: 2, shardSize: 3 }), (d) => {
+      docs = d;
+    });
+    // Shard 1 starts at 1 * shardSize, not at "however many arrived so far".
+    expect(docs.map((d) => d.pos)).toEqual([0, 1, 2, 3, 4]);
+  });
+
+  it("skips a failing shard without losing the rest", async () => {
+    stubFetch({
+      "search-0000.json": rows("A"),
+      "search-0001.json": 404,
+      "search-0002.json": rows("C"),
+    });
+
+    const batches: string[][] = [];
+    await loadSearchShards(
+      info({ shards: 3 }),
+      (docs) => batches.push(docs.map((d) => d.id)),
+      { flushMs: 0 },
+    );
+    expect(batches[batches.length - 1]).toEqual(["A", "C"]);
+  });
+
+  // The documents in a lost shard are gone, but their absence must not shift
+  // everyone else: positions are what address the detail shards, so a hole that
+  // renumbered the survivors would silently point every later document at the
+  // wrong detail shard — and the id check there can only scan the shard it was
+  // sent to, so it could never recover.
+  it("keeps corpus positions correct across a hole left by a failed shard", async () => {
+    stubFetch({
+      "search-0000.json": rows("A1", "A2"),
+      "search-0001.json": 404,
+      "search-0002.json": rows("C1", "C2"),
+    });
+
+    let docs: { id: string; pos?: number }[] = [];
+    await loadSearchShards(info({ shards: 3, shardSize: 2 }), (d) => {
+      docs = d;
+    });
+
+    expect(docs.map((d) => d.id)).toEqual(["A1", "A2", "C1", "C2"]);
+    // C1 is at array index 2 but corpus position 4.
+    expect(docs.map((d) => d.pos)).toEqual([0, 1, 4, 5]);
+  });
+
+  it("retries a shard once before giving up", async () => {
+    let attempts = 0;
+    stubFetch({
+      "search-0000.json": () => {
+        attempts++;
+        return attempts === 1 ? 503 : rows("A1", "A2");
+      },
+    });
+
+    let docs: { id: string }[] = [];
+    await loadSearchShards(info({ shards: 1, shardSize: 2 }), (d) => {
+      docs = d;
+    });
+
+    expect(attempts).toBe(2);
+    expect(docs.map((d) => d.id)).toEqual(["A1", "A2"]);
+  });
+
+  // A 200 whose body is an object rather than an array used to throw inside the
+  // worker, rejecting the whole load and skipping the final forced emit — so
+  // every throttled batch since the last emit was lost, plus an unhandled
+  // rejection.
+  it("survives a shard whose body is valid JSON but not an array", async () => {
+    stubFetch({
+      "search-0000.json": rows("A"),
+      "search-0001.json": { not: "an array" },
+      "search-0002.json": rows("C"),
+    });
+
+    const batches: string[][] = [];
+    await expect(
+      loadSearchShards(
+        info({ shards: 3 }),
+        (docs) => batches.push(docs.map((d) => d.id)),
+        { flushMs: 10_000, now: () => 1_000 },
+      ),
+    ).resolves.toBeUndefined();
+
+    expect(batches[batches.length - 1]).toEqual(["A", "C"]);
+  });
+
+  // A 404 on GitHub Pages returns an HTML error page; without the res.ok check
+  // that would throw inside res.json() and look like a network failure.
+  it("treats a non-OK response as an empty shard", async () => {
+    stubFetch({ "search-0000.json": 500 });
+    const batches: string[][] = [];
+    await loadSearchShards(info({ shards: 1 }), (docs) =>
+      batches.push(docs.map((d) => d.id)),
+    );
+    expect(batches[batches.length - 1]).toEqual([]);
+  });
+
+  it("requests each shard exactly once", async () => {
+    const stub = stubFetch({
+      "search-0000.json": rows("A"),
+      "search-0001.json": rows("B"),
+    });
+    await loadSearchShards(info({ shards: 2 }), () => {});
+    expect(stub.calls.sort()).toEqual(["search-0000.json", "search-0001.json"]);
+  });
+});
+
+describe("createDetailLoader", () => {
+  const detail = (id: string): DetailRecord => ({
+    r: id,
+    abstract: `About ${id}`,
+  });
+
+  const shardedInfo = info({ detailShards: 3, detailShardSize: 2 });
+
+  it("resolves a document by its corpus position", async () => {
+    stubFetch({ "detail-0001.json": [detail("C"), detail("D")] });
+    const load = createDetailLoader(shardedInfo);
+    // index 3 -> shard 1, slot 1
+    await expect(load(3, "D")).resolves.toEqual(detail("D"));
+  });
+
+  it("returns null when detail shards were not built", async () => {
+    const load = createDetailLoader(info({ detailShards: 0 }));
+    await expect(load(0, "A")).resolves.toBeNull();
+  });
+
+  it("returns null for a position beyond the last shard", async () => {
+    const load = createDetailLoader(shardedInfo);
+    await expect(load(999, "Z")).resolves.toBeNull();
+  });
+
+  it("returns null for a null slot (a document with no detail fields)", async () => {
+    stubFetch({ "detail-0000.json": [null, null] });
+    const load = createDetailLoader(shardedInfo);
+    await expect(load(0, "A")).resolves.toBeNull();
+  });
+
+  // Position is derived, not stored, so it has to be checked. Showing another
+  // document's abstract would be worse than showing none.
+  it("falls back to an id scan when the slot holds a different document", async () => {
+    stubFetch({ "detail-0000.json": [detail("WRONG"), detail("A")] });
+    const load = createDetailLoader(shardedInfo);
+    await expect(load(0, "A")).resolves.toEqual(detail("A"));
+  });
+
+  it("returns null when the id is nowhere in the shard", async () => {
+    stubFetch({ "detail-0000.json": [detail("X"), detail("Y")] });
+    const load = createDetailLoader(shardedInfo);
+    await expect(load(0, "A")).resolves.toBeNull();
+  });
+
+  it("fetches a shard once and serves later hits from cache", async () => {
+    const stub = stubFetch({ "detail-0000.json": [detail("A"), detail("B")] });
+    const load = createDetailLoader(shardedInfo);
+
+    await load(0, "A");
+    await load(1, "B");
+
+    expect(stub.calls).toEqual(["detail-0000.json"]);
+  });
+
+  it("shares one request between concurrent opens in the same shard", async () => {
+    const stub = stubFetch({ "detail-0000.json": [detail("A"), detail("B")] });
+    const load = createDetailLoader(shardedInfo);
+
+    const [a, b] = await Promise.all([load(0, "A"), load(1, "B")]);
+
+    expect(stub.calls).toEqual(["detail-0000.json"]);
+    expect(a).toEqual(detail("A"));
+    expect(b).toEqual(detail("B"));
+  });
+
+  it("returns null when the shard request fails", async () => {
+    stubFetch({ "detail-0000.json": 404 });
+    const load = createDetailLoader(shardedInfo);
+    await expect(load(0, "A")).resolves.toBeNull();
+  });
+
+  it("returns null when the shard body is not an array", async () => {
+    stubFetch({ "detail-0000.json": { oops: true } });
+    const load = createDetailLoader(shardedInfo);
+    await expect(load(0, "A")).resolves.toBeNull();
+  });
+});

data/lib/relaton/cli/frontend/src/lib/shards.ts

@@ -0,0 +1,183 @@
+import type {
+  CompactRecord,
+  DetailRecord,
+  IndexDocument,
+  ShardInfo,
+} from "./types";
+
+// Fetching the numbered JSON shards the generator writes beside index.html.
+//
+// Summary shards are loaded progressively so the page is usable long before the
+// whole corpus has arrived; detail shards are fetched one at a time, only when a
+// document panel opens.
+
+/** `search-0007.json` — zero-padded to four digits, matching the generator. */
+export function shardPath(prefix: string, index: number): string {
+  return `${prefix}-${String(index).padStart(4, "0")}.json`;
+}
+
+// A shard that 404s or contains garbage resolves to null rather than throwing,
+// so one bad file can't take down the whole load. Three things are checked:
+// `res.ok`, because a 404 on GitHub Pages returns an HTML error page that would
+// otherwise blow up in res.json(); the parse itself; and that the result is
+// actually an array, because a 200 whose body is `{}` or `5` would otherwise
+// throw at the call site (in a worker, rejecting the whole load).
+async function fetchArray<T>(path: string): Promise<T[] | null> {
+  try {
+    const res = await fetch(path);
+    if (!res.ok) return null;
+    const body = await res.json();
+    return Array.isArray(body) ? (body as T[]) : null;
+  } catch {
+    return null;
+  }
+}
+
+// One retry, because the common failure here is a transient CDN blip mid-deploy
+// and the cost of giving up is a permanent hole in the corpus.
+async function fetchArrayWithRetry<T>(path: string): Promise<T[] | null> {
+  const first = await fetchArray<T>(path);
+  if (first) return first;
+  return fetchArray<T>(path);
+}
+
+export function fromCompact(r: CompactRecord, pos?: number): IndexDocument {
+  return {
+    id: r.r ?? "",
+    title: r.c ?? "",
+    doctype: r.t || null,
+    stage: r.s || null,
+    date: r.d || null,
+    link: r.l || null,
+    yaml: r.u || null,
+    ...(pos === undefined ? {} : { pos }),
+  };
+}
+
+export interface LoadOptions {
+  /** Shards fetched at once. */
+  concurrency?: number;
+  /** Minimum gap between onBatch calls, in ms. */
+  flushMs?: number;
+  /** Injectable clock, for tests. */
+  now?: () => number;
+}
+
+// Load every summary shard, calling `onBatch` with the documents so far.
+//
+// Two properties matter to the caller:
+//
+//   * Order. Records are appended in shard order regardless of the order
+//     responses arrive in, because the corpus order is what the detail shards'
+//     positional lookup is keyed to. A shard that resolves early waits for its
+//     predecessors before being appended.
+//   * Throttling. App.vue re-filters and re-sorts the entire array on every
+//     change, so emitting once per shard would re-sort a growing 166k-element
+//     array dozens of times. Batches are coalesced into one emit per window.
+export async function loadSearchShards(
+  info: ShardInfo,
+  onBatch: (docs: IndexDocument[]) => void,
+  opts: LoadOptions = {},
+): Promise<void> {
+  const total = info.shards;
+  if (total <= 0) return;
+
+  const flushMs = opts.flushMs ?? 750;
+  const now = opts.now ?? (() => Date.now());
+  const slots: (IndexDocument[] | null)[] = new Array(total).fill(null);
+
+  let out: IndexDocument[] = [];
+  let appendedUpTo = 0; // index of the first slot not yet appended to `out`
+  let next = 0;
+  // -Infinity, not 0: the first batch must paint immediately whatever epoch the
+  // clock uses. With 0 it would only do so because Date.now() happens to be huge.
+  let lastEmit = -Infinity;
+
+  const drain = (force: boolean) => {
+    let grew = false;
+    while (appendedUpTo < total && slots[appendedUpTo]) {
+      out = out.concat(slots[appendedUpTo]!);
+      slots[appendedUpTo] = null; // release the intermediate array
+      appendedUpTo++;
+      grew = true;
+    }
+    if (!grew && !force) return;
+
+    const at = now();
+    if (!force && at - lastEmit < flushMs) return;
+    lastEmit = at;
+    onBatch(out); // new array identity each time, so the caller can reassign
+  };
+
+  const worker = async (): Promise<void> => {
+    for (let i = next++; i < total; i = next++) {
+      const rows = await fetchArrayWithRetry<CompactRecord>(
+        shardPath("search", i),
+      );
+      if (!rows) {
+        // The documents are lost, but their absence must not shift anyone
+        // else's corpus position — hence the explicit `pos` below.
+        console.warn(`relaton-index: shard ${shardPath("search", i)} failed to load`);
+        slots[i] = [];
+        drain(false);
+        continue;
+      }
+      // Corpus position is assigned here, from the shard number, so it survives
+      // a hole left by a failed shard and any later reordering of the array.
+      const base = i * info.shardSize;
+      slots[i] = rows.map((row, j) => fromCompact(row, base + j));
+      drain(false);
+    }
+  };
+
+  const workers = Math.max(1, Math.min(opts.concurrency ?? 4, total));
+  await Promise.all(Array.from({ length: workers }, worker));
+  drain(true);
+}
+
+export type DetailLoader = (
+  index: number,
+  id: string,
+) => Promise<DetailRecord | null>;
+
+// Fetch-on-demand loader for the detail shards, with a per-shard cache so
+// paging through neighbouring documents costs one request, not one per
+// document. Concurrent opens within the same shard share a single in-flight
+// request.
+//
+// `index` is the document's position in the full corpus, which is also its
+// position across the detail shards. The entry's `r` is checked against the
+// document id before its fields are used: if the two families ever fell out of
+// step, showing another document's abstract would be worse than showing none,
+// so a mismatch falls back to scanning the shard for the right id.
+export function createDetailLoader(info: ShardInfo): DetailLoader {
+  type Slots = (DetailRecord | null)[];
+  const cache = new Map<number, Slots>();
+  const inFlight = new Map<number, Promise<Slots>>();
+
+  return async function loadDetail(index, id) {
+    const { detailShards, detailShardSize } = info;
+    if (detailShards <= 0 || detailShardSize <= 0 || index < 0) return null;
+
+    const shard = Math.floor(index / detailShardSize);
+    if (shard >= detailShards) return null;
+
+    let entries = cache.get(shard);
+    if (!entries) {
+      let pending = inFlight.get(shard);
+      if (!pending) {
+        pending = fetchArray<DetailRecord | null>(
+          shardPath("detail", shard),
+        ).then((rows) => rows ?? []);
+        inFlight.set(shard, pending);
+      }
+      entries = await pending;
+      cache.set(shard, entries);
+      inFlight.delete(shard);
+    }
+
+    const slot = entries[index % detailShardSize];
+    if (slot && slot.r === id) return slot;
+    return entries.find((entry) => entry != null && entry.r === id) ?? null;
+  };
+}

data/lib/relaton/cli/frontend/src/lib/types.ts

@@ -0,0 +1,148 @@
+/** One contributor (author, publisher, editor, …) on the detail page. */
+export interface Contributor {
+  name: string;
+  role?: string | null;
+}
+
+/** One of a document's identifiers (may repeat per language / scheme). */
+export interface DocIdentifier {
+  id: string;
+  type?: string | null;
+  language?: string | null;
+}
+
+// One related document, e.g. { type: "obsoletedBy", id: "ISO 29862:2024" }.
+// Only the target's DocID is carried — no title or date — because the id is both
+// the label and the key the detail page matches against the rest of the corpus
+// to decide whether it can be rendered as an intra-dataset link.
+export interface Relation {
+  type?: string | null;
+  id: string;
+}
+
+/** One dated event, e.g. { type: "published", value: "2009-06-19" }. */
+export interface DocDate {
+  type?: string | null;
+  value: string;
+}
+
+// One bibliographic document as rendered in the index. This shape is the
+// contract shared with the Ruby side (Relaton::Cli::IndexItemNormalizer):
+// whatever the generator emits as JSON must map here.
+//
+// The first block is the summary contract, carried by the `search-NNNN.json`
+// shards and loaded progressively. The second block holds the richer detail
+// fields, which live in the separate `detail-NNNN.json` shards and are fetched
+// only when a detail panel opens — so they are absent until then, and stay
+// absent for a document that has none.
+export interface IndexDocument {
+  /** Primary rendered Relaton DocID, e.g. "BIPM CCRI 21 (2009)". */
+  id: string;
+  /** Main title (English preferred). */
+  title: string;
+  /** Document type, e.g. "meeting-report" (from ext.doctype). */
+  doctype: string | null;
+  /** Workflow stage, e.g. "published". */
+  stage: string | null;
+  /** Publication date, ISO "YYYY-MM-DD". */
+  date: string | null;
+  /** Human-facing landing/link URL, if any. */
+  link: string | null;
+  /** URL (or relative path) to the raw Relaton YAML source. */
+  yaml: string | null;
+
+  // Assigned by the shard loader, NOT emitted by the generator: this document's
+  // position in the full corpus, which is what addresses its detail shard. It is
+  // carried explicitly rather than derived from the array index because a shard
+  // that fails to load leaves a hole — every later document would then sit at a
+  // lower index than its true position and resolve to the wrong detail shard.
+  pos?: number;
+
+  /** Abstract / summary text (HTML stripped). */
+  abstract?: string | null;
+  /** Edition, e.g. "2". */
+  edition?: string | null;
+  /** Language codes, e.g. ["en", "fr"]. */
+  languages?: string[];
+  /** Subject keywords. */
+  keywords?: string[];
+  /** Publishing organization name. */
+  publisher?: string | null;
+  /** All contributors with their roles. */
+  contributors?: Contributor[];
+  /** All identifiers (multilingual/scheme variants of `id`). */
+  docids?: DocIdentifier[];
+  /** All dated events. */
+  dates?: DocDate[];
+  /** Related documents (obsoletes, updatedBy, partOf, …). */
+  relations?: Relation[];
+}
+
+export interface IndexData {
+  title: string;
+  /** Site blurb shown as the header subtitle (optional; also the meta description). */
+  description?: string | null;
+  documents: IndexDocument[];
+  /** ISO timestamp the site was generated (optional, for the footer). */
+  generated?: string | null;
+}
+
+/** Compact record shape carried by the `search-NNNN.json` summary shards. */
+export interface CompactRecord {
+  r: string; // reference / id
+  c: string; // content / title
+  t?: string | null; // doctype
+  s?: string | null; // stage
+  d?: string | null; // date
+  u?: string | null; // url (yaml)
+  l?: string | null; // link
+}
+
+// One entry in a `detail-NNNN.json` shard: the fields the summary record does
+// NOT carry, tagged with `r` (the document id) so a positional lookup can be
+// verified before its fields are shown. A shard slot is `null` when the
+// document at that position has no detail fields at all — the slot is still
+// written, so position stays aligned with the summary shards.
+export type DetailRecord = Pick<
+  IndexDocument,
+  | "abstract"
+  | "edition"
+  | "languages"
+  | "keywords"
+  | "publisher"
+  | "contributors"
+  | "docids"
+  | "dates"
+  | "relations"
+> & { r: string };
+
+// The shard layout, read from the mount node's data-* attributes. There is
+// deliberately no manifest file: these five scalars are everything the frontend
+// needs, and carrying them inline saves a round-trip before the first shard can
+// be requested (and a failure mode where the manifest 404s but the shards are
+// fine). Shard file names are derived by convention — `search-0000.json`,
+// `detail-0000.json` — and resolve relative to the page, which sits beside them.
+export interface ShardInfo {
+  /** Documents across the whole corpus. */
+  total: number;
+  /** Number of `search-NNNN.json` files. */
+  shards: number;
+  /** Summary records per shard (the last one may hold fewer). */
+  shardSize: number;
+  /** Number of `detail-NNNN.json` files; 0 when built with --no-detail. */
+  detailShards: number;
+  /** Detail slots per shard — the divisor for the positional lookup. */
+  detailShardSize: number;
+}
+
+// Progressive-load state shared with App. A plain reactive object rather than a
+// prop value, because `createApp(App, props)` props are static: reactivity has
+// to come from the object itself.
+export interface Hydration {
+  /** True until every summary shard has landed. */
+  loading: boolean;
+  /** Corpus total, known up front so counts are truthful during loading. */
+  total: number | null;
+  /** Start (or hurry along) the shard load — e.g. when the user searches. */
+  requestAll: () => void;
+}