@eightyfourthousand/data-access 2026.3.0 → 2026.4.0

package/src/lib/glossary/pagination.ts

@@ -0,0 +1,471 @@
+import { DataClient, Passage, PassageDTO, passageFromDTO } from '../types';
+
+type ApiPaginationDirection = 'FORWARD' | 'BACKWARD' | 'AROUND';
+
+type GlossaryTermNode = {
+  uuid: string;
+  authority: string;
+  definition: string | null;
+  termNumber: number;
+  names: {
+    english: string | null;
+    tibetan: string | null;
+    sanskrit: string | null;
+    pali: string | null;
+    chinese: string | null;
+    wylie: string | null;
+    alternatives: string | null;
+  };
+};
+
+type GlossaryTermIndexRow = {
+  glossary_uuid: string;
+  authority_uuid: string;
+  term_number: number | string;
+  definition: string | null;
+  english: string | null;
+  wylie: string | null;
+  tibetan: string | null;
+  sanskrit_plain: string | null;
+  sanskrit_attested: string | null;
+  chinese: string | null;
+  pali: string | null;
+  alternatives: string | null;
+};
+
+type GlossaryPageInfo = {
+  nextCursor: string | null;
+  prevCursor: string | null;
+  hasMoreAfter: boolean;
+  hasMoreBefore: boolean;
+};
+
+export type GlossaryTermConnection = {
+  nodes: GlossaryTermNode[];
+  pageInfo: GlossaryPageInfo;
+  totalCount: number;
+};
+
+export type GlossaryPassagesPage = {
+  items: Passage[];
+  nextCursor: string | null;
+  hasMore: boolean;
+};
+
+const DEFAULT_GLOSSARY_LIMIT = 50;
+const MAX_GLOSSARY_LIMIT = 200;
+const DEFAULT_GLOSSARY_PASSAGES_LIMIT = 10;
+
+function buildGlossaryTermConnection(
+  nodes: GlossaryTermNode[],
+  nextCursor: string | null,
+  prevCursor: string | null,
+  hasMoreAfter: boolean,
+  hasMoreBefore: boolean,
+  totalCount: number,
+): GlossaryTermConnection {
+  return {
+    nodes,
+    pageInfo: {
+      nextCursor,
+      prevCursor,
+      hasMoreAfter,
+      hasMoreBefore,
+    },
+    totalCount,
+  };
+}
+
+function parseOffsetCursor(after?: string) {
+  if (!after) return 0;
+
+  const parsed = Number.parseInt(after, 10);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : 0;
+}
+
+function parseCount(value: number | string | null | undefined) {
+  if (typeof value === 'number') return value;
+
+  const parsed = Number.parseInt(value ?? '', 10);
+  return Number.isFinite(parsed) ? parsed : 0;
+}
+
+function rowToGlossaryTermNode(
+  row: Pick<
+    GlossaryTermIndexRow,
+    | 'glossary_uuid'
+    | 'authority_uuid'
+    | 'definition'
+    | 'term_number'
+    | 'english'
+    | 'wylie'
+    | 'tibetan'
+    | 'sanskrit_plain'
+    | 'sanskrit_attested'
+    | 'chinese'
+    | 'pali'
+    | 'alternatives'
+  > & { withAttestations: boolean },
+): GlossaryTermNode {
+  return {
+    uuid: row.glossary_uuid,
+    authority: row.authority_uuid,
+    definition: row.definition,
+    termNumber: parseCount(row.term_number),
+    names: {
+      english: row.english,
+      alternatives: row.alternatives,
+      wylie: row.wylie,
+      tibetan: row.tibetan,
+      sanskrit: row.withAttestations
+        ? row.sanskrit_attested
+        : row.sanskrit_plain,
+      chinese: row.chinese,
+      pali: row.pali,
+    },
+  };
+}
+
+export const getGlossaryTermPassagesPage = async ({
+  client,
+  uuid,
+  first,
+  after,
+}: {
+  client: DataClient;
+  uuid: string;
+  first?: number;
+  after?: string;
+}): Promise<GlossaryPassagesPage> => {
+  const limit = Math.max(first ?? DEFAULT_GLOSSARY_PASSAGES_LIMIT, 1);
+  const offset = parseOffsetCursor(after);
+
+  const { data: annotations, error: annotationsError } = await client
+    .from('passage_annotations')
+    .select('passage_uuid')
+    .eq('type', 'glossary-instance')
+    .filter('content', 'cs', JSON.stringify([{ uuid }]));
+
+  if (annotationsError) {
+    console.error(
+      'Error fetching glossary passage annotations:',
+      annotationsError,
+    );
+    return { items: [], nextCursor: null, hasMore: false };
+  }
+
+  const passageUuids = (annotations ?? []).map(
+    (annotation: { passage_uuid: string }) => annotation.passage_uuid,
+  );
+
+  if (passageUuids.length === 0) {
+    return { items: [], nextCursor: null, hasMore: false };
+  }
+
+  const { data: passages, error: passagesError } = await client
+    .from('passages')
+    .select('uuid, content, label, sort, type, xmlId, work_uuid, toh')
+    .in('uuid', passageUuids)
+    .order('sort', { ascending: true })
+    .range(offset, offset + limit);
+
+  if (passagesError) {
+    console.error('Error fetching glossary passages:', passagesError);
+    return { items: [], nextCursor: null, hasMore: false };
+  }
+
+  const rows = passages ?? [];
+  const hasMore = rows.length > limit;
+  const items = hasMore ? rows.slice(0, limit) : rows;
+
+  return {
+    items: items.map((passage: PassageDTO) => passageFromDTO(passage)),
+    nextCursor: hasMore ? String(offset + limit) : null,
+    hasMore,
+  };
+};
+
+export const getWorkGlossaryTermsPage = async ({
+  client,
+  workUuid,
+  limit = DEFAULT_GLOSSARY_LIMIT,
+  cursor,
+  direction = 'FORWARD',
+  withAttestations = false,
+}: {
+  client: DataClient;
+  workUuid: string;
+  limit?: number;
+  cursor?: string | null;
+  direction?: ApiPaginationDirection;
+  withAttestations?: boolean;
+}): Promise<GlossaryTermConnection> => {
+  const clampedLimit = Math.min(Math.max(limit, 1), MAX_GLOSSARY_LIMIT);
+
+  if (direction === 'AROUND') {
+    return getWorkGlossaryTermsAround({
+      client,
+      workUuid,
+      limit: clampedLimit,
+      cursor,
+      withAttestations,
+    });
+  }
+
+  const [
+    { count, error: countError },
+    { data: cursorRows, error: cursorError },
+  ] = await Promise.all([
+    client
+      .from('glossary_term_index')
+      .select('authority_uuid', { count: 'exact', head: true })
+      .eq('work_uuid', workUuid),
+    cursor
+      ? client
+          .from('glossary_term_index')
+          .select('authority_uuid, term_number')
+          .eq('work_uuid', workUuid)
+          .eq('authority_uuid', cursor)
+          .limit(1)
+      : Promise.resolve({ data: [], error: null }),
+  ]);
+
+  if (countError) {
+    console.error('Error counting glossary terms:', countError);
+    return buildGlossaryTermConnection([], null, null, false, false, 0);
+  }
+
+  if (cursorError) {
+    console.error('Error fetching glossary cursor term:', cursorError);
+    return buildGlossaryTermConnection([], null, null, false, false, 0);
+  }
+
+  const totalCount = count ?? 0;
+  if (totalCount === 0) {
+    return buildGlossaryTermConnection([], null, null, false, false, 0);
+  }
+
+  const cursorRow = cursor
+    ? ((cursorRows ?? [])[0] as
+        | { authority_uuid: string; term_number: number | string }
+        | undefined)
+    : undefined;
+
+  if (cursor && !cursorRow) {
+    return buildGlossaryTermConnection(
+      [],
+      null,
+      null,
+      false,
+      false,
+      totalCount,
+    );
+  }
+
+  const cursorTermNumber =
+    cursorRow !== undefined ? parseCount(cursorRow.term_number) : null;
+  let query = client
+    .from('glossary_term_index')
+    .select(
+      `glossary_uuid,
+       authority_uuid,
+       term_number,
+       definition,
+       english,
+       wylie,
+       tibetan,
+       sanskrit_plain,
+       sanskrit_attested,
+       chinese,
+       pali,
+       alternatives`,
+    )
+    .eq('work_uuid', workUuid);
+
+  if (cursorTermNumber !== null) {
+    query =
+      direction === 'FORWARD'
+        ? query.gt('term_number', cursorTermNumber)
+        : query.lt('term_number', cursorTermNumber);
+  }
+
+  const ascending = direction === 'FORWARD';
+  const { data, error } = await query
+    .order('term_number', { ascending })
+    .limit(clampedLimit);
+
+  if (error) {
+    console.error('Error fetching paginated glossary terms:', error);
+    return buildGlossaryTermConnection(
+      [],
+      null,
+      null,
+      false,
+      false,
+      totalCount,
+    );
+  }
+
+  const pageRows = (data ?? []) as GlossaryTermIndexRow[];
+  const rows = ascending ? pageRows : [...pageRows].reverse();
+  if (rows.length === 0) {
+    return buildGlossaryTermConnection(
+      [],
+      null,
+      null,
+      false,
+      false,
+      totalCount,
+    );
+  }
+
+  const nodes = rows.map((row) =>
+    rowToGlossaryTermNode({ ...row, withAttestations }),
+  );
+  const firstRow = rows[0];
+  const lastRow = rows[rows.length - 1];
+  const hasMoreBefore = parseCount(firstRow.term_number) > 1;
+  const hasMoreAfter = parseCount(lastRow.term_number) < totalCount;
+
+  return buildGlossaryTermConnection(
+    nodes,
+    hasMoreAfter ? lastRow.authority_uuid : null,
+    hasMoreBefore ? firstRow.authority_uuid : null,
+    hasMoreAfter,
+    hasMoreBefore,
+    totalCount,
+  );
+};
+
+export const getWorkGlossaryTermsAround = async ({
+  client,
+  workUuid,
+  limit,
+  cursor,
+  withAttestations,
+}: {
+  client: DataClient;
+  workUuid: string;
+  limit: number;
+  cursor?: string | null;
+  withAttestations: boolean;
+}): Promise<GlossaryTermConnection> => {
+  if (!cursor) {
+    return getWorkGlossaryTermsPage({
+      client,
+      workUuid,
+      limit,
+      cursor: null,
+      direction: 'FORWARD',
+      withAttestations,
+    });
+  }
+
+  const [
+    { count, error: countError },
+    { data: cursorRows, error: cursorError },
+  ] = await Promise.all([
+    client
+      .from('glossary_term_index')
+      .select('authority_uuid', { count: 'exact', head: true })
+      .eq('work_uuid', workUuid),
+    client
+      .from('glossary_term_index')
+      .select('authority_uuid, term_number')
+      .eq('work_uuid', workUuid)
+      .eq('authority_uuid', cursor)
+      .limit(1),
+  ]);
+
+  if (countError) {
+    console.error('Error counting glossary terms:', countError);
+    return buildGlossaryTermConnection([], null, null, false, false, 0);
+  }
+
+  if (cursorError) {
+    console.error('Error fetching glossary cursor term:', cursorError);
+    return getWorkGlossaryTermsPage({
+      client,
+      workUuid,
+      limit,
+      cursor: null,
+      direction: 'FORWARD',
+      withAttestations,
+    });
+  }
+
+  const totalCount = count ?? 0;
+  const cursorRow = (cursorRows ?? [])[0] as
+    | { authority_uuid: string; term_number: number | string }
+    | undefined;
+
+  if (!cursorRow) {
+    return getWorkGlossaryTermsPage({
+      client,
+      workUuid,
+      limit,
+      cursor: null,
+      direction: 'FORWARD',
+      withAttestations,
+    });
+  }
+
+  const cursorTermNumber = parseCount(cursorRow.term_number);
+  let startTerm = Math.max(1, cursorTermNumber - Math.floor(limit / 2));
+  let endTerm = startTerm + limit - 1;
+
+  if (endTerm > totalCount) {
+    endTerm = totalCount;
+    startTerm = Math.max(1, endTerm - limit + 1);
+  }
+
+  const { data, error } = await client
+    .from('glossary_term_index')
+    .select(
+      `glossary_uuid,
+       authority_uuid,
+       term_number,
+       definition,
+       english,
+       wylie,
+       tibetan,
+       sanskrit_plain,
+       sanskrit_attested,
+       chinese,
+       pali`,
+    )
+    .eq('work_uuid', workUuid)
+    .gte('term_number', startTerm)
+    .lte('term_number', endTerm)
+    .order('term_number', { ascending: true });
+
+  if (error) {
+    console.error('Error fetching glossary terms around cursor:', error);
+    return buildGlossaryTermConnection(
+      [],
+      null,
+      null,
+      false,
+      false,
+      totalCount,
+    );
+  }
+
+  const rows = (data ?? []) as GlossaryTermIndexRow[];
+  const nodes = rows.map((row) =>
+    rowToGlossaryTermNode({ ...row, withAttestations }),
+  );
+  const hasMoreBefore = startTerm > 1;
+  const hasMoreAfter = endTerm < totalCount;
+  const firstRow = rows[0];
+  const lastRow = rows[rows.length - 1];
+
+  return buildGlossaryTermConnection(
+    nodes,
+    hasMoreAfter && lastRow ? lastRow.authority_uuid : null,
+    hasMoreBefore && firstRow ? firstRow.authority_uuid : null,
+    hasMoreAfter,
+    hasMoreBefore,
+    totalCount,
+  );
+};

package/src/lib/passage/batch.ts

@@ -0,0 +1,229 @@
+import {
+  AlignmentDTO,
+  AnnotationDTO,
+  DataClient,
+  Passage,
+  PassageDTO,
+  Passages,
+  passageFromDTO,
+} from '../types';
+
+export const getAnnotationsByPassageUuids = async ({
+  client,
+  passageUuids,
+}: {
+  client: DataClient;
+  passageUuids: readonly string[];
+}): Promise<Map<string, AnnotationDTO[]>> => {
+  const annotationsByPassage = new Map<string, AnnotationDTO[]>();
+
+  if (passageUuids.length === 0) {
+    return annotationsByPassage;
+  }
+
+  const pageSize = 1000;
+  let allData: AnnotationDTO[] = [];
+  let offset = 0;
+  let hasMore = true;
+
+  while (hasMore) {
+    const { data, error } = await client
+      .from('passage_annotations')
+      .select('uuid, passage_uuid, type, start, end, content, toh')
+      .in('passage_uuid', passageUuids as string[])
+      .not('type', 'like', 'deprecated%')
+      .range(offset, offset + pageSize - 1);
+
+    if (error) {
+      console.error('Error batch loading annotations:', error);
+      return new Map();
+    }
+
+    allData = allData.concat((data ?? []) as AnnotationDTO[]);
+    hasMore = (data?.length ?? 0) === pageSize;
+    offset += pageSize;
+  }
+
+  for (const annotation of allData) {
+    const passageUuid = annotation.passage_uuid;
+    if (!passageUuid) continue;
+    const existing = annotationsByPassage.get(passageUuid);
+    if (existing) {
+      existing.push(annotation);
+    } else {
+      annotationsByPassage.set(passageUuid, [annotation]);
+    }
+  }
+
+  for (const annotations of annotationsByPassage.values()) {
+    annotations.sort((a, b) => {
+      if (a.start !== b.start) return a.start - b.start;
+      return b.end - a.end;
+    });
+  }
+
+  return annotationsByPassage;
+};
+
+export const getAlignmentsByPassageUuids = async ({
+  client,
+  passageUuids,
+}: {
+  client: DataClient;
+  passageUuids: readonly string[];
+}): Promise<Map<string, AlignmentDTO[]>> => {
+  const alignmentsByPassage = new Map<string, AlignmentDTO[]>();
+
+  if (passageUuids.length === 0) {
+    return alignmentsByPassage;
+  }
+
+  const pageSize = 1000;
+  let allData: (AlignmentDTO & { passage_uuid: string })[] = [];
+  let offset = 0;
+  let hasMore = true;
+
+  while (hasMore) {
+    const { data, error } = await client
+      .from('passage_alignments')
+      .select(
+        'passage_uuid, folio_uuid, toh, tibetan, folio_number, volume_number',
+      )
+      .in('passage_uuid', passageUuids as string[])
+      .range(offset, offset + pageSize - 1);
+
+    if (error) {
+      console.error('Error batch loading alignments:', error);
+      return new Map();
+    }
+
+    allData = allData.concat((data ?? []) as AlignmentDTO[]);
+    hasMore = (data?.length ?? 0) === pageSize;
+    offset += pageSize;
+  }
+
+  for (const row of allData) {
+    const existing = alignmentsByPassage.get(row.passage_uuid);
+    if (existing) {
+      existing.push(row);
+    } else {
+      alignmentsByPassage.set(row.passage_uuid, [row]);
+    }
+  }
+
+  return alignmentsByPassage;
+};
+
+export const getPassageLabelsByUuids = async ({
+  client,
+  passageUuids,
+}: {
+  client: DataClient;
+  passageUuids: readonly string[];
+}): Promise<Map<string, string>> => {
+  const labelsByUuid = new Map<string, string>();
+
+  if (passageUuids.length === 0) {
+    return labelsByUuid;
+  }
+
+  const { data, error } = await client
+    .from('passages')
+    .select('uuid, label')
+    .in('uuid', passageUuids as string[]);
+
+  if (error) {
+    console.error('Error batch loading passage labels:', error);
+    return labelsByUuid;
+  }
+
+  for (const passage of data ?? []) {
+    if (passage.label) {
+      labelsByUuid.set(passage.uuid, passage.label);
+    }
+  }
+
+  return labelsByUuid;
+};
+
+export const getPassageReferencesByTargetUuids = async ({
+  client,
+  passageUuids,
+}: {
+  client: DataClient;
+  passageUuids: readonly string[];
+}): Promise<Map<string, Passages>> => {
+  const referencesByTargetUuid = new Map<string, Passages>();
+
+  if (passageUuids.length === 0) {
+    return referencesByTargetUuid;
+  }
+
+  const { data: annotations, error: annotationsError } = await client.rpc(
+    'get_passage_annotations_by_content_uuids',
+    {
+      annotation_type: 'end-note-link',
+      target_uuids: passageUuids as string[],
+    },
+  );
+
+  if (annotationsError) {
+    console.error(
+      'Error batch loading passage reference annotations:',
+      annotationsError,
+    );
+    return referencesByTargetUuid;
+  }
+
+  const targetToSourceUuids = new Map<string, Set<string>>();
+  const allSourceUuids = new Set<string>();
+
+  for (const row of (annotations ?? []) as Array<{
+    passage_uuid: string;
+    target_uuid: string;
+  }>) {
+    if (!row.target_uuid || !row.passage_uuid) continue;
+    allSourceUuids.add(row.passage_uuid);
+    const sourceSet =
+      targetToSourceUuids.get(row.target_uuid) ?? new Set<string>();
+    sourceSet.add(row.passage_uuid);
+    targetToSourceUuids.set(row.target_uuid, sourceSet);
+  }
+
+  const sourceUuidArray = Array.from(allSourceUuids);
+  const passageMap = new Map<string, Passage>();
+  const inBatchSize = 300;
+
+  for (let i = 0; i < sourceUuidArray.length; i += inBatchSize) {
+    const batch = sourceUuidArray.slice(i, i + inBatchSize);
+    const { data, error } = await client
+      .from('passages')
+      .select('uuid, content, label, sort, type, toh, xmlId, work_uuid')
+      .in('uuid', batch);
+
+    if (error) {
+      console.error('Error batch loading passage reference data:', error);
+      return new Map();
+    }
+
+    for (const row of (data ?? []) as PassageDTO[]) {
+      passageMap.set(row.uuid, passageFromDTO(row));
+    }
+  }
+
+  for (const targetUuid of passageUuids) {
+    const sourceUuids = targetToSourceUuids.get(targetUuid);
+    if (!sourceUuids) continue;
+    const references: Passages = [];
+    for (const sourceUuid of sourceUuids) {
+      const passage = passageMap.get(sourceUuid);
+      if (passage) references.push(passage);
+    }
+    referencesByTargetUuid.set(
+      targetUuid,
+      references.sort((a, b) => a.sort - b.sort),
+    );
+  }
+
+  return referencesByTargetUuid;
+};

package/src/lib/passage/index.ts

@@ -0,0 +1,5 @@
+export * from './batch';
+export * from './pagination';
+export * from './read';
+export * from './replace-persistence';
+export * from './save';