glossarist 0.3.8 → 0.4.1

package/src/models/index.d.ts

@@ -65,6 +65,7 @@ export class Concept extends GlossaristModel {
   definition(lang: string): string | null;
   setLocalization(lang: string, lc: LocalizedConcept | Record<string, unknown>): this;
   hasLocalization(lang: string): boolean;
+  walkTexts(): Iterable<{ text: string; source: string }>;
   static fromJSON(data: Record<string, unknown>): Concept;
 }
 
@@ -96,6 +97,7 @@ export class LocalizedConcept extends GlossaristModel {
   readonly related: RelatedConcept[];
   readonly primaryDesignation: string | null;
   readonly primaryDefinition: string | null;
+  walkTexts(basePath: string): Iterable<{ text: string; source: string }>;
   static fromJSON(data: Record<string, unknown>): LocalizedConcept;
 }
 
@@ -243,6 +245,9 @@ export class ConceptDate extends GlossaristModel {
 export class DetailedDefinition extends GlossaristModel {
   readonly content: string;
   readonly sources: ConceptSource[];
+  readonly examples: DetailedDefinition[];
+  walkTexts(path: string): Iterable<{ text: string; source: string }>;
+  static fromJSON(data: Record<string, unknown>): DetailedDefinition;
 }
 
 export class NonVerbRep extends GlossaristModel {

package/src/models/index.js

@@ -12,9 +12,20 @@ export { DesignationRelationship, DESIGNATION_RELATIONSHIP_TYPES } from './desig
 export { ConceptReference } from './concept-reference.js';
 export { ConceptDate, DATE_TYPES } from './concept-date.js';
 export { DetailedDefinition } from './detailed-definition.js';
-export { NonVerbRep } from './non-verb-rep.js';
 export { Pronunciation } from './pronunciation.js';
 export { GrammarInfo, GRAMMAR_GENDERS, GRAMMAR_NUMBERS, GRAMMAR_PARTS_OF_SPEECH } from './grammar-info.js';
 export { Locality } from './locality.js';
 export { GcrMetadata } from './gcr-metadata.js';
 export { GcrStatistics } from './gcr-statistics.js';
+export { RegistrableModel } from './registrable.js';
+export { NonVerbalEntity } from './non-verbal-entity.js';
+export { SharedNonVerbalEntity } from './shared-non-verbal-entity.js';
+export { Figure, FigureImage } from './figure.js';
+export { Table } from './table.js';
+export { Formula } from './formula.js';
+export { NonVerbRep, NON_VERBAL_TYPES } from './non-verb-rep.js';
+export { NonVerbalReference } from './non-verbal-reference.js';
+export { FigureReference, TableReference, FormulaReference } from './non-verbal-references.js';
+export { BibliographyEntry } from './bibliography-entry.js';
+export { BibliographyData } from './bibliography-data.js';
+export { fetchLocalizedString, localizedStringIsEmpty, localizedStringIsPresent } from './localized-string.js';

package/src/models/localized-concept.js

@@ -99,6 +99,27 @@ export class LocalizedConcept extends GlossaristModel {
     return this.definitions[0]?.content ?? null;
   }
 
+  *_textSlots() {
+    yield ['definitions', this.definitions];
+    yield ['notes', this.notes];
+    yield ['examples', this.examples];
+    yield ['annotations', this.annotations];
+  }
+
+  /**
+   * Yield every content-text fragment in this localization, recursing
+   * through nested examples. `basePath` prefixes every emitted
+   * `source` path; pass `localizations.<lang>` to get paths consistent
+   * with the rest of the codebase. Designations are not included.
+   */
+  *walkTexts(basePath) {
+    for (const [name, arr] of this._textSlots()) {
+      for (let i = 0; i < arr.length; i++) {
+        yield* arr[i].walkTexts(`${basePath}.${name}[${i}]`);
+      }
+    }
+  }
+
   toJSON() {
     const obj = {};
     if (this.languageCode) obj.language_code = this.languageCode;

package/src/models/localized-string.js

@@ -0,0 +1,19 @@
+const FALLBACK_LANG = 'eng';
+
+export function fetchLocalizedString(hash, lang, fallback = FALLBACK_LANG) {
+  if (hash == null || typeof hash !== 'object') return null;
+  const direct = hash[lang] ?? hash[String(lang)];
+  if (direct != null) return direct;
+  if (fallback != null && fallback !== lang) {
+    return hash[fallback] ?? hash[String(fallback)] ?? null;
+  }
+  return null;
+}
+
+export function localizedStringIsEmpty(hash) {
+  return hash == null || (typeof hash === 'object' && Object.keys(hash).length === 0);
+}
+
+export function localizedStringIsPresent(hash) {
+  return !localizedStringIsEmpty(hash);
+}

package/src/models/non-verb-rep.js

@@ -1,23 +1,25 @@
-import { GlossaristModel } from './base.js';
-import { ConceptSource } from './concept-source.js';
+import { NonVerbalEntity } from './non-verbal-entity.js';
+import { FigureImage } from './figure.js';
 
-export class NonVerbRep extends GlossaristModel {
+export const NON_VERBAL_TYPES = Object.freeze(['image', 'table', 'formula']);
+
+export class NonVerbRep extends NonVerbalEntity {
   constructor(data = {}) {
-    super();
+    super(data);
     this.type = data.type ?? null;
-    this.ref = data.ref ?? null;
-    this.text = data.text ?? null;
-    this.sources = (data.sources ?? []).map(
-      s => s instanceof ConceptSource ? s : new ConceptSource(s)
-    );
+    this._rawImages = data.images ?? [];
+    this._images = null;
+  }
+
+  get images() {
+    return this._lazy('_images', '_rawImages',
+      i => i instanceof FigureImage ? i : new FigureImage(i));
   }
 
   toJSON() {
-    const obj = {};
+    const obj = super.toJSON();
     if (this.type != null) obj.type = this.type;
-    if (this.ref != null) obj.ref = this.ref;
-    if (this.text != null) obj.text = this.text;
-    if (this.sources.length > 0) obj.sources = this.sources.map(s => s.toJSON());
+    this._serialize(obj, 'images', '_images', '_rawImages');
     return obj;
   }
 

package/src/models/non-verbal-entity.js

@@ -0,0 +1,39 @@
+import { RegistrableModel } from './registrable.js';
+import { ConceptSource } from './concept-source.js';
+
+export class NonVerbalEntity extends RegistrableModel {
+  constructor(data = {}) {
+    super();
+    this.caption = data.caption ?? null;
+    this.description = data.description ?? null;
+    this.alt = data.alt ?? null;
+    this._rawSources = data.sources ?? [];
+    this._sources = null;
+  }
+
+  get sources() {
+    return this._lazy('_sources', '_rawSources',
+      s => s instanceof ConceptSource ? s : new ConceptSource(s));
+  }
+
+  findById(_targetId) {
+    return null;
+  }
+
+  allIds() {
+    return [];
+  }
+
+  toJSON() {
+    const obj = {};
+    if (this.caption != null) obj.caption = this.caption;
+    if (this.description != null) obj.description = this.description;
+    if (this.alt != null) obj.alt = this.alt;
+    this._serialize(obj, 'sources', '_sources', '_rawSources');
+    return obj;
+  }
+
+  static fromJSON(data) {
+    return NonVerbalEntity.fromData(data);
+  }
+}

package/src/models/non-verbal-reference.js

@@ -0,0 +1,35 @@
+import { RegistrableModel } from './registrable.js';
+
+export class NonVerbalReference extends RegistrableModel {
+  constructor(data = {}) {
+    super();
+    this.entityId = data.entityId ?? data.entity_id ?? data.ref ?? data.id ?? null;
+    this.display = data.display ?? null;
+  }
+
+  get dedupKey() {
+    return [this.constructor.name, this.entityId];
+  }
+
+  toJSON() {
+    if (this.display != null) {
+      return { ref: this.entityId, display: this.display };
+    }
+    return this.entityId;
+  }
+
+  static fromJSON(data) {
+    if (data instanceof NonVerbalReference) return data;
+    if (typeof data === 'string') {
+      return new NonVerbalReference({ entityId: data });
+    }
+    const entityId = data.entityId ?? data.entity_id ?? data.ref ?? data.id ?? null;
+    const display = data.display ?? null;
+    const type = data.type;
+    if (type && this._registry().has(type)) {
+      const Cls = this._registry().get(type);
+      return new Cls({ entityId, display });
+    }
+    return new NonVerbalReference({ entityId, display });
+  }
+}

package/src/models/non-verbal-references.js

@@ -0,0 +1,16 @@
+import { NonVerbalReference } from './non-verbal-reference.js';
+
+export class FigureReference extends NonVerbalReference {
+  static fromJSON(data) { return NonVerbalReference.fromJSON(data); }
+}
+NonVerbalReference.register('figure', FigureReference);
+
+export class TableReference extends NonVerbalReference {
+  static fromJSON(data) { return NonVerbalReference.fromJSON(data); }
+}
+NonVerbalReference.register('table', TableReference);
+
+export class FormulaReference extends NonVerbalReference {
+  static fromJSON(data) { return NonVerbalReference.fromJSON(data); }
+}
+NonVerbalReference.register('formula', FormulaReference);

package/src/models/registrable.js

@@ -0,0 +1,25 @@
+import { GlossaristModel } from './base.js';
+
+const _registries = new WeakMap();
+
+export class RegistrableModel extends GlossaristModel {
+  static _registry() {
+    let map = _registries.get(this);
+    if (!map) {
+      map = new Map();
+      _registries.set(this, map);
+    }
+    return map;
+  }
+
+  static register(type, cls) {
+    this._registry().set(type, cls);
+  }
+
+  static fromData(data) {
+    if (data instanceof this) return data;
+    const type = data?.type;
+    const Cls = type ? this._registry().get(type) ?? this : this;
+    return new Cls(data);
+  }
+}

package/src/models/shared-non-verbal-entity.js

@@ -0,0 +1,28 @@
+import { NonVerbalEntity } from './non-verbal-entity.js';
+
+export class SharedNonVerbalEntity extends NonVerbalEntity {
+  constructor(data = {}) {
+    super(data);
+    this.id = data.id ?? null;
+    this.identifier = data.identifier ?? null;
+  }
+
+  findById(targetId) {
+    return this.id === targetId ? this : null;
+  }
+
+  allIds() {
+    return this.id != null ? [this.id] : [];
+  }
+
+  toJSON() {
+    const obj = super.toJSON();
+    if (this.id != null) obj.id = this.id;
+    if (this.identifier != null) obj.identifier = this.identifier;
+    return obj;
+  }
+
+  static fromJSON(data) {
+    return SharedNonVerbalEntity.fromData(data);
+  }
+}

package/src/models/table.js

@@ -0,0 +1,21 @@
+import { SharedNonVerbalEntity } from './shared-non-verbal-entity.js';
+import { NonVerbalEntity } from './non-verbal-entity.js';
+
+export class Table extends SharedNonVerbalEntity {
+  constructor(data = {}) {
+    super(data);
+    this.content = data.content ?? null;
+    this.format = data.format ?? null;
+  }
+
+  toJSON() {
+    const obj = super.toJSON();
+    if (this.content != null) obj.content = this.content;
+    if (this.format != null) obj.format = this.format;
+    return obj;
+  }
+
+  static fromJSON(data) { return new Table(data); }
+}
+
+NonVerbalEntity.register('table', Table);

package/src/reference-mention.js

@@ -27,6 +27,12 @@
 
 const NUMERIC_RE = /^\d+(?:[.-]\d+)+$/;
 
+const NVR_PREFIXES = Object.freeze([
+  { prefix: 'fig:', kind: 'fig-ref' },
+  { prefix: 'table:', kind: 'table-ref' },
+  { prefix: 'formula:', kind: 'formula-ref' },
+]);
+
 /**
  * Parse the body of a {{...}} mention (without the braces).
  *
@@ -63,7 +69,17 @@ export function parseMention(raw) {
     };
   }
 
-  // 3. Comma-separated form: {{id, render}}.
+  // 3. NVR prefixes (fig:/table:/formula:) — config-driven dispatch.
+  for (const { prefix, kind } of NVR_PREFIXES) {
+    const escPrefix = prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const match = body.match(new RegExp(`^${escPrefix}([^,}]+)(?:,(.*))?$`));
+    if (match) {
+      const label = match[2] !== undefined ? unquoteLabel(match[2].trim()) : null;
+      return { kind, key: match[1].trim(), label, raw: body };
+    }
+  }
+
+  // 4. Comma-separated form: {{id, render}}.
   //    ID always comes first, render text always comes last.
   const commaIdx = body.indexOf(',');
   if (commaIdx !== -1) {
@@ -76,12 +92,12 @@ export function parseMention(raw) {
     return { kind: 'designation', id, label, raw: body };
   }
 
-  // 4. Bare numeric id.
+  // 5. Bare numeric id.
   if (NUMERIC_RE.test(body)) {
     return { kind: 'numeric', id: body, label: null, raw: body };
   }
 
-  // 5. Anything else is unresolved at the parse layer.
+  // 6. Anything else is unresolved at the parse layer.
   return { kind: 'unresolved', raw: body };
 }
 

package/src/reference-resolver.js

@@ -13,6 +13,21 @@ export class Reference {
     this.resolution = extras.resolution ?? null;
     this.lookupKey = extras.lookupKey ?? null;
   }
+
+  get dedupKey() {
+    if (this.type === 'bibliography') {
+      return ['bibliography',
+        this.sourceId ?? this.citation?.ref?.id ?? this.target];
+    }
+    if (this.type === 'figure' || this.type === 'table' || this.type === 'formula') {
+      return [this.type, this.lookupKey?.entityId ?? this.target];
+    }
+    if (this.type === 'concept') {
+      return ['concept',
+        this.lookupKey?.id ?? this.lookupKey?.designation ?? this.target];
+    }
+    return [this.type, this.target];
+  }
 }
 
 function refTarget(rc) {
@@ -33,6 +48,18 @@ export function resolveBibliographyRecord(citationRef, registry) {
   return bioColl.byId(citationRef.id);
 }
 
+export function findNonVerbalEntity(ref, registry) {
+  const { entityType, entityId } = ref.lookupKey ?? {};
+  if (!entityType || !entityId) return null;
+  const collection = registry[`nvr:${entityType}`];
+  if (!collection) return null;
+  for (const entity of collection) {
+    const found = entity.findById(entityId);
+    if (found) return found;
+  }
+  return null;
+}
+
 export class ReferenceResolver {
   extractReferences(concept) {
     const refs = [];
@@ -46,6 +73,22 @@ export class ReferenceResolver {
       }
     }
 
+    for (const ref of concept.figures) {
+      refs.push(new Reference('figure', ref.display ?? ref.entityId, 'structural', 'figures', {
+        lookupKey: { entityType: 'figure', entityId: ref.entityId },
+      }));
+    }
+    for (const ref of concept.tables) {
+      refs.push(new Reference('table', ref.display ?? ref.entityId, 'structural', 'tables', {
+        lookupKey: { entityType: 'table', entityId: ref.entityId },
+      }));
+    }
+    for (const ref of concept.formulas) {
+      refs.push(new Reference('formula', ref.display ?? ref.entityId, 'structural', 'formulas', {
+        lookupKey: { entityType: 'formula', entityId: ref.entityId },
+      }));
+    }
+
     for (const lang of concept.languages) {
       const lc = concept.localization(lang);
       if (!lc) continue;
@@ -58,44 +101,24 @@ export class ReferenceResolver {
         }
       }
 
-      const texts = this._collectTexts(lc, lang);
-      for (const { text, source } of texts) {
+      for (const { text, source } of lc.walkTexts(`localizations.${lang}`)) {
         for (const ref of this._extractFromText(text, source, concept)) {
           refs.push(ref);
         }
       }
     }
 
-    return refs;
+    return this._dedup(refs);
   }
 
-  _collectTexts(lc, lang) {
-    const out = [];
-    for (let i = 0; i < lc.definitions.length; i++) {
-      const content = lc.definitions[i]?.content;
-      if (typeof content === 'string') {
-        out.push({ text: content, source: `localizations.${lang}.definitions[${i}].content` });
-      }
-    }
-    for (let i = 0; i < lc.notes.length; i++) {
-      const content = lc.notes[i]?.content;
-      if (content) {
-        out.push({ text: content, source: `localizations.${lang}.notes[${i}].content` });
-      }
-    }
-    for (let i = 0; i < lc.examples.length; i++) {
-      const content = lc.examples[i]?.content;
-      if (typeof content === 'string') {
-        out.push({ text: content, source: `localizations.${lang}.examples[${i}].content` });
-      }
-    }
-    for (let i = 0; i < lc.annotations.length; i++) {
-      const content = lc.annotations[i]?.content;
-      if (typeof content === 'string') {
-        out.push({ text: content, source: `localizations.${lang}.annotations[${i}].content` });
-      }
-    }
-    return out;
+  _dedup(refs) {
+    const seen = new Set();
+    return refs.filter(ref => {
+      const key = JSON.stringify(ref.dedupKey);
+      if (seen.has(key)) return false;
+      seen.add(key);
+      return true;
+    });
   }
 
   _extractFromText(text, source, concept) {
@@ -114,6 +137,21 @@ export class ReferenceResolver {
             resolution: null,
           }));
           break;
+        case 'fig-ref':
+          refs.push(new Reference('figure', parsed.label ?? parsed.key, 'embedded', source, {
+            lookupKey: { entityType: 'figure', entityId: parsed.key },
+          }));
+          break;
+        case 'table-ref':
+          refs.push(new Reference('table', parsed.label ?? parsed.key, 'embedded', source, {
+            lookupKey: { entityType: 'table', entityId: parsed.key },
+          }));
+          break;
+        case 'formula-ref':
+          refs.push(new Reference('formula', parsed.label ?? parsed.key, 'embedded', source, {
+            lookupKey: { entityType: 'formula', entityId: parsed.key },
+          }));
+          break;
         case 'numeric':
           refs.push(new Reference('concept', parsed.label ?? parsed.id, 'embedded', source, {
             lookupKey: { id: parsed.id },
@@ -168,6 +206,9 @@ export class ReferenceResolver {
     switch (ref.type) {
       case 'concept':      return this._resolveConcept(ref, registry);
       case 'bibliography': return this._resolveBibliography(ref, registry);
+      case 'figure':
+      case 'table':
+      case 'formula':      return this._resolveNonVerbal(ref, registry);
       case 'dataset':      return this._resolveDataset(ref, registry);
       case 'typed-ref':    return this._resolveTypedRef(ref, registry);
       case 'standard':     return this._resolveStandard(ref, registry);
@@ -175,6 +216,10 @@ export class ReferenceResolver {
     }
   }
 
+  _resolveNonVerbal(ref, registry) {
+    return findNonVerbalEntity(ref, registry);
+  }
+
   _resolveConcept(ref, registry) {
     if (ref.lookupKey?.id) {
       const dataset = ref.lookupKey.dataset;
@@ -218,7 +263,8 @@ export class ReferenceResolver {
   resolveAll(concept, registry) {
     const resolved = new Map();
     for (const ref of this.extractReferences(concept)) {
-      if (ref.type === 'concept' || ref.type === 'bibliography') {
+      if (ref.type === 'concept' || ref.type === 'bibliography'
+          || ref.type === 'figure' || ref.type === 'table' || ref.type === 'formula') {
         const target = this.resolveReference(ref, registry);
         if (target != null) {
           const key = ref.target ?? ref.uri ?? ref.sourceId;

package/src/render-classification.js

@@ -1,4 +1,4 @@
-import { resolveBibliographyRecord } from './reference-resolver.js';
+import { resolveBibliographyRecord, findNonVerbalEntity } from './reference-resolver.js';
 
 export class ReferenceClassifier {
   constructor(registry = {}, sourceDatasetId = null, options = {}) {
@@ -14,6 +14,9 @@ export class ReferenceClassifier {
       case 'concept':      return this._classifyConcept(ref);
       case 'dataset':      return this._classifyDataset(ref);
       case 'bibliography': return this._classifyBibliography(ref);
+      case 'figure':
+      case 'table':
+      case 'formula':      return this._classifyNonVerbal(ref);
       case 'typed-ref':    return this._classifyTypedRef(ref);
       case 'standard':     return 'legacy-standard';
       default:             return 'unknown';
@@ -63,4 +66,10 @@ export class ReferenceClassifier {
   _classifyTypedRef(_ref) {
     return 'typed-ref';
   }
+
+  _classifyNonVerbal(ref) {
+    return findNonVerbalEntity(ref, this.registry)
+      ? 'internal-citation'
+      : 'external-citation';
+  }
 }

package/src/validators/asset-index.js

@@ -0,0 +1,66 @@
+import { DATASET_ASSETS } from '../dataset-asset.js';
+
+export class AssetIndex {
+  constructor() {
+    this._paths = new Set();
+  }
+
+  get paths() {
+    return [...this._paths].sort();
+  }
+
+  get size() {
+    return this._paths.size;
+  }
+
+  register(path) {
+    if (path == null) return;
+    this._paths.add(this._normalize(path));
+  }
+
+  has(path) {
+    if (path == null) return false;
+    return this._paths.has(this._normalize(path));
+  }
+
+  [Symbol.iterator]() {
+    return this._paths[Symbol.iterator]();
+  }
+
+  _normalize(path) {
+    return String(path).replace(/^\//, '');
+  }
+
+  static async fromGcrPackage(pkg) {
+    const index = new AssetIndex();
+    const names = await pkg.imageFileNames();
+    for (const name of names) {
+      index.register(name);
+    }
+    return index;
+  }
+
+  static fromDirectory(datasetPath, fs) {
+    const index = new AssetIndex();
+    const imagesAsset = DATASET_ASSETS.find(
+      a => a.type === 'directory' && a.path === 'images');
+    if (!imagesAsset) return index;
+
+    const imagesDir = `${datasetPath}/${imagesAsset.path}`;
+    _walkDir(fs, imagesDir, imagesAsset.path, index);
+    return index;
+  }
+}
+
+function _walkDir(fs, dirPath, relativePrefix, index) {
+  if (!fs.existsSync(dirPath)) return;
+  for (const entry of fs.readdirSync(dirPath, { withFileTypes: true })) {
+    const fullPath = `${dirPath}/${entry.name}`;
+    const relPath = `${relativePrefix}/${entry.name}`;
+    if (entry.isDirectory()) {
+      _walkDir(fs, fullPath, relPath, index);
+    } else {
+      index.register(relPath);
+    }
+  }
+}

package/src/validators/index.js

@@ -14,6 +14,8 @@ export {
   UuidFormatRule,
   SourceUrnFormatRule,
   CiteRefIntegrityRule,
+  NonVerbalRefIntegrityRule,
+  OrphanedImagesRule,
 } from './v3-rules.js';
 
 import { ConceptValidator, LanguageCodeRule, DesignationTypeRule, EntryStatusRule } from './concept-validator.js';
@@ -29,6 +31,7 @@ import {
   UuidFormatRule,
   SourceUrnFormatRule,
   CiteRefIntegrityRule,
+  NonVerbalRefIntegrityRule,
 } from './v3-rules.js';
 
 const _default = new ConceptValidator()
@@ -43,7 +46,8 @@ const _default = new ConceptValidator()
   .addRule(new UuidFormatRule())
   .addRule(new SourceUrnFormatRule())
   .addRule(new RelationshipTypeRule())
-  .addRule(new CiteRefIntegrityRule());
+  .addRule(new CiteRefIntegrityRule())
+  .addRule(new NonVerbalRefIntegrityRule());
 
 export function validateConcept(concept) {
   return _default.validate(concept);