iev 0.4.6 → 0.4.8

data/lib/iev/bibliography_builder.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Iev
+  # Builds a `Glossarist::BibliographyData` from the sources cited across a
+  # concept collection.
+  #
+  # Each unique `(source, id)` pair from a concept's `ConceptSource#origin`
+  # becomes one `BibliographyEntry`. The entry's `id` is the normalized
+  # anchor that `Glossarist::Validation::BibliographyIndex` uses for
+  # resolution — so the same normalization rules are applied here.
+  module BibliographyBuilder
+    module_function
+
+    # @param concepts [Enumerable<Glossarist::ManagedConcept>]
+    # @return [Glossarist::BibliographyData]
+    def build(concepts)
+      entries = collect_entries(concepts)
+      Glossarist::BibliographyData.new(entries: entries)
+    end
+
+    def collect_entries(concepts)
+      seen = {}
+      concepts.each do |concept|
+        concept.localizations.each do |l10n|
+          collect_from_l10n(l10n, seen)
+        end
+        Array(concept.sources).each { |src| add_source_entry(src, seen) }
+      end
+      seen.values.sort_by(&:id)
+    end
+    private_class_method :collect_entries
+
+    def collect_from_l10n(l10n, seen)
+      Array(l10n.all_sources).each { |src| add_source_entry(src, seen) }
+    end
+    private_class_method :collect_from_l10n
+
+    def add_source_entry(source, seen)
+      ref = source_origin_ref(source)
+      return unless ref
+
+      seen[entry_label(ref)] ||= build_entry(ref, source&.origin)
+    end
+    private_class_method :add_source_entry
+
+    def source_origin_ref(source)
+      ref = source&.origin&.ref
+      return unless ref&.source && !ref.source.strip.empty?
+
+      ref
+    end
+    private_class_method :source_origin_ref
+
+    def build_entry(ref, origin)
+      label = entry_label(ref)
+      Glossarist::BibliographyEntry.new(
+        id: normalize_anchor(label),
+        reference: label,
+        link: origin&.link,
+        type: type_for(ref.source),
+      )
+    end
+    private_class_method :build_entry
+
+    def entry_label(ref)
+      [ref.source, ref.id].compact.join(" ").strip
+    end
+    private_class_method :entry_label
+
+    # Mirrors `Glossarist::Validation::BibliographyIndex#normalize_anchor`
+    # so the id we write matches what the validator will look up.
+    def normalize_anchor(anchor)
+      anchor.to_s.gsub(/[ \/:]/, "_").gsub(/__+/, "_").downcase
+    end
+    private_class_method :normalize_anchor
+
+    def type_for(source)
+      case source.to_s
+      when /\A(IEV|VIM|JCGM)/ then "vocabulary"
+      when /\AITU/ then "recommendation"
+      when /\A(BIPM|BBIPM)/ then "brochure"
+      else "standard"
+      end
+    end
+    private_class_method :type_for
+  end
+end

data/lib/iev/cli/command.rb

@@ -187,9 +187,38 @@ module Iev
         exit 1
       end
 
+      desc "reparse", "Parse cached HTML into Glossarist YAML concept files"
+      option :output, aliases: :o,
+                      default: File.join(Dir.pwd, "concepts"),
+                      desc: "Output directory"
+      option :pages_dir, desc: "Directory containing cached HTML pages"
+      def reparse
+        pages_dir = options[:pages_dir] || Iev.config.pages_dir
+        concepts_dir = build_reparse_dir(options[:output])
+        collection = Glossarist::ManagedConceptCollection.new
+        Dir.glob(File.join(pages_dir, "pages", "*.html")).sort.each do |path|
+          code = File.basename(path, ".html")
+          html = File.read(path, encoding: "utf-8")
+          doc = Nokogiri::HTML(html)
+          raw = Iev::Scraper::PageParser.new(doc, code).parse
+          next unless raw
+          collection.store(build_concept_from_raw(code, raw))
+        end
+        Iev::MultiDocYaml.save_grouped_concepts(collection, concepts_dir.to_s)
+        info "Reparsed #{collection.count} concepts into #{concepts_dir}."
+      end
+
       def self.exit_on_failure?
         true
       end
+
+      private
+
+      def build_reparse_dir(output)
+        concepts_dir = Pathname.new(output).expand_path.join("concepts")
+        FileUtils.mkdir_p(concepts_dir)
+        concepts_dir
+      end
     end
   end
 end

data/lib/iev/cli/command_helper.rb

@@ -121,12 +121,7 @@ module Iev
       end
 
       def print_concept_grouped_yaml(concept)
-        content = []
-        content << concept.to_yaml
-        concept.localized_concepts.each_key do |lang|
-          content << concept.localization(lang).to_yaml
-        end
-        puts content.join("\n")
+        puts Iev::MultiDocYaml.join(Iev::MultiDocYaml.parts_for(concept))
       end
     end
   end

data/lib/iev/config.rb

@@ -6,12 +6,21 @@ module Iev
   class Config
     DEFAULT_REMOTE_BASE_URL = "https://raw.githubusercontent.com/glossarist/glossarist-data-iev/main/concepts"
 
-    attr_accessor :data_path, :cache_dir, :remote_base_url
+    # Default mirror cache lives OUTSIDE the repo, at <repo-parent>/iev-data-latest,
+    # which is itself a private git repo (glossarist/iev-data-latest) so the
+    # raw HTML snapshots are versioned and shareable. This keeps the cache
+    # outside repo operations (rm -rf, branch switches), persists across
+    # reboots (so Dir.tmpdir is wrong — macOS purges it), and survives
+    # accidental deletion of the iev working tree.
+    DEFAULT_PAGES_DIR = File.expand_path("../../../iev-data-latest", __dir__)
+
+    attr_accessor :data_path, :cache_dir, :remote_base_url, :pages_dir
 
     def initialize
       @data_path = ENV.fetch("IEV_DATA_PATH", nil)
       @cache_dir = ENV["IEV_CACHE_DIR"] || File.join(Dir.tmpdir, "iev-cache")
       @remote_base_url = DEFAULT_REMOTE_BASE_URL
+      @pages_dir = ENV["IEV_PAGES_DIR"] || DEFAULT_PAGES_DIR
     end
   end
 end

data/lib/iev/exporter.rb

@@ -56,13 +56,18 @@ module Iev
       collection = build_collection(dataset)
       add_subject_area_concepts(collection) if @include_areas
       build_section_narrower_relations(collection) if @include_areas
+      figures = FigureBuilder.extract!(collection)
+      enrich_references(collection)
       save_collection(collection)
+      save_figures(figures)
+      save_bibliography(BibliographyBuilder.build(collection))
       save_register
       elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
 
       @stats = {
         concept_count: collection.count,
         localized_count: localized_count(collection),
+        figure_count: figures.length,
         elapsed_seconds: elapsed,
       }
       collection
@@ -171,8 +176,35 @@ module Iev
 
     def save_collection(collection)
       concepts_dir = output_dir.expand_path.join("concepts")
-      FileUtils.mkdir_p(concepts_dir)
-      collection.save_grouped_concepts_to_files(concepts_dir.to_s)
+      MultiDocYaml.save_grouped_concepts(collection, concepts_dir.to_s)
+    end
+
+    def save_figures(figures)
+      return if figures.empty?
+
+      figures_dir = output_dir.expand_path.join("figures")
+      FileUtils.mkdir_p(figures_dir)
+      figures.each do |figure|
+        path = figures_dir.join("#{figure.id}.yaml")
+        File.write(path, figure.to_yaml, encoding: "utf-8")
+      end
+      puts "Written #{figures.length} figures to figures/" if $stdout.tty?
+    end
+
+    def save_bibliography(bibliography)
+      return if bibliography.entries.empty?
+
+      path = output_dir.expand_path.join("bibliography.yaml")
+      FileUtils.mkdir_p(path.dirname)
+      File.write(path, bibliography.to_yaml, encoding: "utf-8")
+      count = bibliography.entries.length
+      puts "Written bibliography.yaml with #{count} entries" if $stdout.tty?
+    end
+
+    def enrich_references(collection)
+      return if collection.none?
+
+      Glossarist::ConceptEnricher.new.inject_references(collection.to_a)
     end
 
     def save_register
@@ -275,7 +307,8 @@ module Iev
 
       concept.related << Glossarist::RelatedConcept.new(
         type: "broader",
-        content: code.section_uri,
+        # lang-keyed hash per glossarist 2.13 RelatedConcept L10N invariant
+        content: { "eng" => code.section_uri },
         ref: Glossarist::ConceptRef.new(source: "IEV", id: code.section_uri),
       )
     end
@@ -300,7 +333,7 @@ module Iev
         narrower = child_ids.sort.map do |child_id|
           Glossarist::RelatedConcept.new(
             type: "narrower",
-            content: child_id,
+            content: { "eng" => child_id },
             ref: Glossarist::ConceptRef.new(source: "IEV", id: child_id),
           )
         end

data/lib/iev/figure_builder.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Iev
+  # Hoists IEV figure references into dataset-shared Figure entities.
+  #
+  # IEV source data carries figures as inline SIMG tags, which Utilities
+  # rewrites to AsciiDoc image macros (+image::/assets/images/parts/{area}/
+  # FILE[Figure N - caption]+). This builder walks every concept's
+  # localizations, finds those image macros, promotes each to a
+  # dataset-shared Glossarist::Figure entity, and rewrites the inline text
+  # to a V3 figure mention (+{{fig:id, display}}+).
+  #
+  # The Figure entity is shared across concepts and languages — captions
+  # from different localizations merge into the same {lang => text} hash.
+  # The structural link from concept to figure is a FigureReference entry
+  # on ManagedConceptData#figures.
+  #
+  # Extraction is destructive: it mutates DetailedDefinition#content and
+  # appends FigureReference entries. Returns the unique Figure entities so
+  # the exporter can persist them to figures/{id}.yaml.
+  module FigureBuilder
+    # URL path prefix emitted by Utilities when converting SIMG tags.
+    # Kept in sync with Utilities::IMAGE_PATH_PREFIX (without the macro).
+    PATH_PREFIX = "/assets/images/parts"
+    private_constant :PATH_PREFIX
+
+    # Matches AsciiDoc image macros emitted by Utilities#process_simg_figures.
+    IMAGE_MACRO_REGEX = /
+      image::#{Regexp.escape(PATH_PREFIX)}
+      \/(?<area>\d+)\/(?<file>[\w.-]+)\[(?<caption>[^\]]*)\]
+    /x
+
+    # Captures "Figure N" label and the trailing caption text.
+    CAPTION_REGEX = /\A(?<label>Figure\s+\d+)\s*[–-]\s*(?<text>.+)\z/m
+
+    module_function
+
+    # @param collection [Glossarist::ManagedConceptCollection]
+    # @return [Array<Glossarist::Figure>] unique figures, sorted by id
+    def extract!(collection)
+      figures_by_id = {}
+
+      collection.each do |concept|
+        concept.localizations.each do |l10n|
+          process_localization(l10n, concept, figures_by_id)
+        end
+      end
+
+      figures_by_id.values.sort_by(&:id)
+    end
+
+    def process_localization(l10n, concept, figures_by_id)
+      lang = l10n.data&.language_code
+      return unless lang && lang.length == 3
+
+      Glossarist::ConceptData.detailed_definition_fields.each do |field|
+        process_field(l10n, field, lang, concept, figures_by_id)
+      end
+    end
+    private_class_method :process_localization
+
+    def process_field(l10n, field, lang, concept, figures_by_id)
+      l10n.data.public_send(field).each do |dd|
+        next unless dd.content&.include?("image::")
+
+        rewritten, hits = extract_from_text(dd.content, lang)
+        next if hits.empty?
+
+        dd.content = rewritten
+        hits.each { |hit| register_figure(hit, concept, figures_by_id) }
+      end
+    end
+    private_class_method :process_field
+
+    # @return [Array<(String, Array<Hash>)>] rewritten text and per-match
+    #   figure descriptors ({ id:, identifier:, caption:, lang:, image: })
+    def extract_from_text(text, lang)
+      hits = []
+      rewritten = text.gsub(IMAGE_MACRO_REGEX) do
+        hit = build_hit(Regexp.last_match, lang)
+        hits << hit
+        mention_for(hit)
+      end
+      [rewritten, hits]
+    end
+    private_class_method :extract_from_text
+
+    def build_hit(match, lang)
+      identifier, caption = parse_caption(match[:caption])
+      {
+        id: figure_id_for(match[:file]),
+        identifier: identifier,
+        caption: caption,
+        lang: lang,
+        image: build_image(match[:area], match[:file]),
+      }
+    end
+    private_class_method :build_hit
+
+    def build_image(area, file)
+      Glossarist::FigureImage.new(
+        src: "#{PATH_PREFIX}/#{area}/#{file}",
+        format: format_for(file),
+      )
+    end
+    private_class_method :build_image
+
+    def parse_caption(bracket)
+      stripped = bracket.to_s.strip
+      return [nil, nil] if stripped.empty?
+
+      if (m = stripped.match(CAPTION_REGEX))
+        label = m[:label].gsub(/\s+/, " ")
+        [label, m[:text].strip]
+      elsif stripped.match?(/\AFigure\s+\d+\z/)
+        [stripped, nil]
+      else
+        [nil, stripped]
+      end
+    end
+    private_class_method :parse_caption
+
+    def figure_id_for(file)
+      "fig-#{file.sub(/\.[^.]+\z/, '')}"
+    end
+    private_class_method :figure_id_for
+
+    def format_for(file)
+      File.extname(file).delete_prefix(".").downcase
+    end
+    private_class_method :format_for
+
+    def mention_for(hit)
+      parts = [hit[:identifier], hit[:caption]].compact
+      return "{{fig:#{hit[:id]}}}" if parts.empty?
+
+      "{{fig:#{hit[:id]}, #{parts.join(' - ')}}}"
+    end
+    private_class_method :mention_for
+
+    # Add or merge a figure descriptor into the shared index, and ensure the
+    # concept carries a FigureReference to it.
+    def register_figure(hit, concept, figures_by_id)
+      figure = figures_by_id[hit[:id]] ||= build_figure(hit)
+      merge_caption!(figure, hit)
+      add_image_if_missing(figure, hit[:image])
+      add_figure_reference(concept, hit[:id], hit[:identifier])
+    end
+    private_class_method :register_figure
+
+    def build_figure(hit)
+      Glossarist::Figure.new(
+        id: hit[:id],
+        identifier: hit[:identifier],
+        images: [],
+        caption: {},
+      )
+    end
+    private_class_method :build_figure
+
+    def merge_caption!(figure, hit)
+      return unless hit[:caption]
+
+      figure.caption ||= {}
+      figure.caption[hit[:lang]] ||= hit[:caption]
+    end
+    private_class_method :merge_caption!
+
+    def add_image_if_missing(figure, image)
+      return if figure.images.any? { |i| i.src == image.src }
+
+      figure.images << image
+    end
+    private_class_method :add_image_if_missing
+
+    def add_figure_reference(concept, figure_id, display)
+      refs = Array(concept.data.figures)
+      return if refs.any? { |r| r.entity_id == figure_id }
+
+      concept.data.figures = refs + [
+        Glossarist::FigureReference.new(entity_id: figure_id, display: display),
+      ]
+    end
+    private_class_method :add_figure_reference
+  end
+end

data/lib/iev/multi_doc_yaml.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Iev
+  # Assembles serialized glossarist models into valid multi-document YAML.
+  #
+  # Lutaml::Model versions below 0.8.30 emit `to_yaml` without the leading
+  # "---" document start marker. Concatenating such outputs naively (as
+  # glossarist's `save_grouped_concepts_to_files` does with `join("\n")`)
+  # merges all documents into one YAML mapping, with later keys overriding
+  # earlier ones — the last localized concept would silently replace the
+  # managed concept's data. `join` guarantees each part is a self-contained
+  # YAML document and is a no-op for outputs that already carry the marker.
+  module MultiDocYaml
+    DOCUMENT_START = "---"
+
+    module_function
+
+    # Serialized YAML documents for a managed concept: the concept first,
+    # then each of its localizations (glossarist's grouped file layout).
+    def parts_for(concept)
+      [concept.to_yaml].tap do |parts|
+        (concept.localized_concepts || {}).each_key do |lang|
+          l10n = concept.localization(lang)
+          parts << l10n.to_yaml if l10n
+        end
+      end
+    end
+
+    # Concatenates serialized YAML parts into a parseable document stream.
+    def join(parts)
+      parts.map { |part| with_document_start(part) }.join("\n")
+    end
+
+    # Writes joined parts to +path+ as UTF-8.
+    def write(path, parts)
+      File.write(path, join(parts), encoding: "utf-8")
+    end
+
+    # Writes one grouped YAML file per managed concept (concept followed by
+    # its localizations) into +dir+, named by the concept's uuid — the same
+    # layout and naming as glossarist's `save_grouped_concepts_to_files`,
+    # but document-marker-safe.
+    def save_grouped_concepts(collection, dir)
+      FileUtils.mkdir_p(dir)
+
+      collection.each do |concept|
+        write(File.join(dir.to_s, "#{concept.uuid}.yaml"), parts_for(concept))
+      end
+    end
+
+    # Prepends the YAML document start marker unless already present.
+    def with_document_start(yaml)
+      return yaml if yaml.start_with?(DOCUMENT_START)
+
+      "#{DOCUMENT_START}\n#{yaml}"
+    end
+  end
+end

data/lib/iev/reconciler/change.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # A single field-level difference between the termbase snapshot and
+    # the live snapshot of one concept. Collected into a ChangeSet.
+    #
+    # @attr code [String] IEV code (e.g. "102-01-01")
+    # @attr field [Symbol] what changed: :designation, :definition, :notes,
+    #   :examples, :status, :language_added, :language_removed
+    # @attr language [String, nil] ISO 639-2 code, nil for concept-level
+    # @attr old_value [String, nil] termbase value (nil for additions)
+    # @attr new_value [String, nil] live value (nil for removals)
+    # @attr detected_at [String] ISO 8601 date the change was observed
+    Change = Struct.new(:code, :field, :language, :old_value, :new_value,
+                        :detected_at, keyword_init: true) do
+      def to_h
+        {
+          code: code,
+          field: field.to_s,
+          language: language,
+          detected_at: detected_at,
+          old_value: old_value,
+          new_value: new_value,
+        }
+      end
+    end
+  end
+end

data/lib/iev/reconciler/change_set.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # A collection of Change records for a single concept. Enumerable,
+    # with query helpers for filtering by field or language.
+    class ChangeSet
+      include Enumerable
+
+      attr_reader :code
+
+      def initialize(code)
+        @code = code
+        @changes = []
+      end
+
+      def add(change)
+        @changes << change
+        self
+      end
+
+      def each(&block)
+        @changes.each(&block)
+      end
+
+      def size
+        @changes.size
+      end
+
+      def empty?
+        @changes.empty?
+      end
+
+      def by_field(field)
+        @changes.select { |c| c.field == field }
+      end
+
+      def by_language(lang)
+        @changes.select { |c| c.language == lang }
+      end
+
+      # Hash of field => count, for summary statistics.
+      def summary
+        @changes.group_by(&:field).transform_values(&:size)
+      end
+    end
+  end
+end

data/lib/iev/reconciler/concept_merger.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "date"
+
+module Iev
+  module Reconciler
+    # Merges a termbase concept and a live concept into a single
+    # ReconciledConcept. The merge preserves lifecycle dates from the
+    # termbase, uses live content for current values, and records any
+    # detected changes as ConceptDate amendments.
+    #
+    # The termbase concept is mutated in place — the pipeline streams
+    # through codes one at a time, so each concept object is used once
+    # and discarded. This avoids the cost of cloning 20k+ model objects.
+    class ConceptMerger
+      # @param differ [ContentDiffer] injectable for testing
+      def initialize(differ: ContentDiffer.new)
+        @differ = differ
+      end
+
+      # @param code [String] IEV code
+      # @param termbase_concept [Glossarist::ManagedConcept, nil]
+      # @param live_concept [Glossarist::ManagedConcept, nil]
+      # @param detected_at [String] ISO 8601 date
+      # @return [ReconciledConcept]
+      def merge(code:, termbase_concept:, live_concept:, detected_at:)
+        if termbase_concept && live_concept
+          merge_both(code, termbase_concept, live_concept, detected_at)
+        elsif termbase_concept
+          retire(code, termbase_concept, detected_at)
+        else
+          ReconciledConcept.new(
+            managed_concept: live_concept,
+            change_set: ChangeSet.new(code),
+            source: :live_only,
+          )
+        end
+      end
+
+      private
+
+      def merge_both(code, tb_concept, live_concept, detected_at)
+        change_set = @differ.diff(tb_concept, live_concept, detected_at: detected_at)
+
+        merge_localized_content(tb_concept, live_concept)
+
+        unless change_set.empty?
+          add_date(tb_concept, "amended", detected_at)
+          flag_stale_translations(tb_concept, change_set, detected_at)
+        end
+
+        ReconciledConcept.new(
+          managed_concept: tb_concept,
+          change_set: change_set,
+          source: :both,
+        )
+      end
+
+      SOURCE_LANGUAGES = %w[eng fra].freeze
+
+      # When source-language (English/French) notes or definitions change,
+      # the translations in other languages are potentially stale because
+      # they haven't been updated to match. Flag them with an annotation.
+      def flag_stale_translations(concept, change_set, detected_at)
+        source_changes = change_set.select do |c|
+          SOURCE_LANGUAGES.include?(c.language) &&
+            %i[definition notes].include?(c.field)
+        end
+        return if source_changes.empty?
+
+        stale_langs = (concept.localized_concepts&.keys || []) - SOURCE_LANGUAGES
+        return if stale_langs.empty?
+
+        source_fields = source_changes.map { |c| "#{c.language}:#{c.field}" }.uniq.join(", ")
+        stale_langs.each do |lang|
+          lc = concept.localization(lang)
+          next unless lc&.data
+          lc.data.examples ||= []
+          lc.data.examples << Glossarist::DetailedDefinition.new(
+            content: "[Reconciler] Translation may be stale — source language content changed (#{source_fields}) on #{detected_at}",
+          )
+        end
+      end
+
+      def retire(code, tb_concept, detected_at)
+        tb_concept.status = "retired"
+        add_date(tb_concept, "retired", detected_at)
+
+        cs = ChangeSet.new(code)
+        cs.add(Change.new(
+          code: code, field: :status, language: nil,
+          old_value: tb_concept.status, new_value: "retired",
+          detected_at: detected_at,
+        ))
+
+        ReconciledConcept.new(
+          managed_concept: tb_concept,
+          change_set: cs,
+          source: :termbase_only,
+        )
+      end
+
+      def merge_localized_content(merged, live_concept)
+        live_concept.localized_concepts&.each_key do |lang|
+          live_lc = live_concept.localization(lang)
+          next unless live_lc&.data
+
+          existing = merged.localization(lang)
+          if existing&.data
+            update_from_live(existing.data, live_lc.data)
+          else
+            merged.add_l10n(live_lc)
+          end
+        end
+      end
+
+      def update_from_live(target_data, live_data)
+        target_data.definition = live_data.definition if live_data.definition&.any?
+        target_data.notes = live_data.notes if live_data.notes&.any?
+        target_data.examples = live_data.examples if live_data.examples&.any?
+        target_data.terms = live_data.terms if live_data.terms&.any?
+      end
+
+      def add_date(concept, type, date)
+        concept.dates ||= []
+        concept.dates << Glossarist::ConceptDate.new(type: type, date: date)
+      end
+    end
+  end
+end