iev 0.4.7 → 0.4.8

data/lib/iev/cli/command.rb

@@ -187,9 +187,38 @@ module Iev
         exit 1
       end
 
+      desc "reparse", "Parse cached HTML into Glossarist YAML concept files"
+      option :output, aliases: :o,
+                      default: File.join(Dir.pwd, "concepts"),
+                      desc: "Output directory"
+      option :pages_dir, desc: "Directory containing cached HTML pages"
+      def reparse
+        pages_dir = options[:pages_dir] || Iev.config.pages_dir
+        concepts_dir = build_reparse_dir(options[:output])
+        collection = Glossarist::ManagedConceptCollection.new
+        Dir.glob(File.join(pages_dir, "pages", "*.html")).sort.each do |path|
+          code = File.basename(path, ".html")
+          html = File.read(path, encoding: "utf-8")
+          doc = Nokogiri::HTML(html)
+          raw = Iev::Scraper::PageParser.new(doc, code).parse
+          next unless raw
+          collection.store(build_concept_from_raw(code, raw))
+        end
+        Iev::MultiDocYaml.save_grouped_concepts(collection, concepts_dir.to_s)
+        info "Reparsed #{collection.count} concepts into #{concepts_dir}."
+      end
+
       def self.exit_on_failure?
         true
       end
+
+      private
+
+      def build_reparse_dir(output)
+        concepts_dir = Pathname.new(output).expand_path.join("concepts")
+        FileUtils.mkdir_p(concepts_dir)
+        concepts_dir
+      end
     end
   end
 end

data/lib/iev/cli/command_helper.rb

@@ -121,12 +121,7 @@ module Iev
       end
 
       def print_concept_grouped_yaml(concept)
-        content = []
-        content << concept.to_yaml
-        concept.localized_concepts.each_key do |lang|
-          content << concept.localization(lang).to_yaml
-        end
-        puts content.join("\n")
+        puts Iev::MultiDocYaml.join(Iev::MultiDocYaml.parts_for(concept))
       end
     end
   end

data/lib/iev/config.rb

@@ -6,12 +6,21 @@ module Iev
   class Config
     DEFAULT_REMOTE_BASE_URL = "https://raw.githubusercontent.com/glossarist/glossarist-data-iev/main/concepts"
 
-    attr_accessor :data_path, :cache_dir, :remote_base_url
+    # Default mirror cache lives OUTSIDE the repo, at <repo-parent>/iev-data-latest,
+    # which is itself a private git repo (glossarist/iev-data-latest) so the
+    # raw HTML snapshots are versioned and shareable. This keeps the cache
+    # outside repo operations (rm -rf, branch switches), persists across
+    # reboots (so Dir.tmpdir is wrong — macOS purges it), and survives
+    # accidental deletion of the iev working tree.
+    DEFAULT_PAGES_DIR = File.expand_path("../../../iev-data-latest", __dir__)
+
+    attr_accessor :data_path, :cache_dir, :remote_base_url, :pages_dir
 
     def initialize
       @data_path = ENV.fetch("IEV_DATA_PATH", nil)
       @cache_dir = ENV["IEV_CACHE_DIR"] || File.join(Dir.tmpdir, "iev-cache")
       @remote_base_url = DEFAULT_REMOTE_BASE_URL
+      @pages_dir = ENV["IEV_PAGES_DIR"] || DEFAULT_PAGES_DIR
     end
   end
 end

data/lib/iev/exporter.rb

@@ -176,8 +176,7 @@ module Iev
 
     def save_collection(collection)
       concepts_dir = output_dir.expand_path.join("concepts")
-      FileUtils.mkdir_p(concepts_dir)
-      collection.save_grouped_concepts_to_files(concepts_dir.to_s)
+      MultiDocYaml.save_grouped_concepts(collection, concepts_dir.to_s)
     end
 
     def save_figures(figures)
@@ -308,7 +307,8 @@ module Iev
 
       concept.related << Glossarist::RelatedConcept.new(
         type: "broader",
-        content: code.section_uri,
+        # lang-keyed hash per glossarist 2.13 RelatedConcept L10N invariant
+        content: { "eng" => code.section_uri },
         ref: Glossarist::ConceptRef.new(source: "IEV", id: code.section_uri),
       )
     end
@@ -333,7 +333,7 @@ module Iev
         narrower = child_ids.sort.map do |child_id|
           Glossarist::RelatedConcept.new(
             type: "narrower",
-            content: child_id,
+            content: { "eng" => child_id },
             ref: Glossarist::ConceptRef.new(source: "IEV", id: child_id),
           )
         end

data/lib/iev/multi_doc_yaml.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Iev
+  # Assembles serialized glossarist models into valid multi-document YAML.
+  #
+  # Lutaml::Model versions below 0.8.30 emit `to_yaml` without the leading
+  # "---" document start marker. Concatenating such outputs naively (as
+  # glossarist's `save_grouped_concepts_to_files` does with `join("\n")`)
+  # merges all documents into one YAML mapping, with later keys overriding
+  # earlier ones — the last localized concept would silently replace the
+  # managed concept's data. `join` guarantees each part is a self-contained
+  # YAML document and is a no-op for outputs that already carry the marker.
+  module MultiDocYaml
+    DOCUMENT_START = "---"
+
+    module_function
+
+    # Serialized YAML documents for a managed concept: the concept first,
+    # then each of its localizations (glossarist's grouped file layout).
+    def parts_for(concept)
+      [concept.to_yaml].tap do |parts|
+        (concept.localized_concepts || {}).each_key do |lang|
+          l10n = concept.localization(lang)
+          parts << l10n.to_yaml if l10n
+        end
+      end
+    end
+
+    # Concatenates serialized YAML parts into a parseable document stream.
+    def join(parts)
+      parts.map { |part| with_document_start(part) }.join("\n")
+    end
+
+    # Writes joined parts to +path+ as UTF-8.
+    def write(path, parts)
+      File.write(path, join(parts), encoding: "utf-8")
+    end
+
+    # Writes one grouped YAML file per managed concept (concept followed by
+    # its localizations) into +dir+, named by the concept's uuid — the same
+    # layout and naming as glossarist's `save_grouped_concepts_to_files`,
+    # but document-marker-safe.
+    def save_grouped_concepts(collection, dir)
+      FileUtils.mkdir_p(dir)
+
+      collection.each do |concept|
+        write(File.join(dir.to_s, "#{concept.uuid}.yaml"), parts_for(concept))
+      end
+    end
+
+    # Prepends the YAML document start marker unless already present.
+    def with_document_start(yaml)
+      return yaml if yaml.start_with?(DOCUMENT_START)
+
+      "#{DOCUMENT_START}\n#{yaml}"
+    end
+  end
+end

data/lib/iev/reconciler/change.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # A single field-level difference between the termbase snapshot and
+    # the live snapshot of one concept. Collected into a ChangeSet.
+    #
+    # @attr code [String] IEV code (e.g. "102-01-01")
+    # @attr field [Symbol] what changed: :designation, :definition, :notes,
+    #   :examples, :status, :language_added, :language_removed
+    # @attr language [String, nil] ISO 639-2 code, nil for concept-level
+    # @attr old_value [String, nil] termbase value (nil for additions)
+    # @attr new_value [String, nil] live value (nil for removals)
+    # @attr detected_at [String] ISO 8601 date the change was observed
+    Change = Struct.new(:code, :field, :language, :old_value, :new_value,
+                        :detected_at, keyword_init: true) do
+      def to_h
+        {
+          code: code,
+          field: field.to_s,
+          language: language,
+          detected_at: detected_at,
+          old_value: old_value,
+          new_value: new_value,
+        }
+      end
+    end
+  end
+end

data/lib/iev/reconciler/change_set.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # A collection of Change records for a single concept. Enumerable,
+    # with query helpers for filtering by field or language.
+    class ChangeSet
+      include Enumerable
+
+      attr_reader :code
+
+      def initialize(code)
+        @code = code
+        @changes = []
+      end
+
+      def add(change)
+        @changes << change
+        self
+      end
+
+      def each(&block)
+        @changes.each(&block)
+      end
+
+      def size
+        @changes.size
+      end
+
+      def empty?
+        @changes.empty?
+      end
+
+      def by_field(field)
+        @changes.select { |c| c.field == field }
+      end
+
+      def by_language(lang)
+        @changes.select { |c| c.language == lang }
+      end
+
+      # Hash of field => count, for summary statistics.
+      def summary
+        @changes.group_by(&:field).transform_values(&:size)
+      end
+    end
+  end
+end

data/lib/iev/reconciler/concept_merger.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "date"
+
+module Iev
+  module Reconciler
+    # Merges a termbase concept and a live concept into a single
+    # ReconciledConcept. The merge preserves lifecycle dates from the
+    # termbase, uses live content for current values, and records any
+    # detected changes as ConceptDate amendments.
+    #
+    # The termbase concept is mutated in place — the pipeline streams
+    # through codes one at a time, so each concept object is used once
+    # and discarded. This avoids the cost of cloning 20k+ model objects.
+    class ConceptMerger
+      # @param differ [ContentDiffer] injectable for testing
+      def initialize(differ: ContentDiffer.new)
+        @differ = differ
+      end
+
+      # @param code [String] IEV code
+      # @param termbase_concept [Glossarist::ManagedConcept, nil]
+      # @param live_concept [Glossarist::ManagedConcept, nil]
+      # @param detected_at [String] ISO 8601 date
+      # @return [ReconciledConcept]
+      def merge(code:, termbase_concept:, live_concept:, detected_at:)
+        if termbase_concept && live_concept
+          merge_both(code, termbase_concept, live_concept, detected_at)
+        elsif termbase_concept
+          retire(code, termbase_concept, detected_at)
+        else
+          ReconciledConcept.new(
+            managed_concept: live_concept,
+            change_set: ChangeSet.new(code),
+            source: :live_only,
+          )
+        end
+      end
+
+      private
+
+      def merge_both(code, tb_concept, live_concept, detected_at)
+        change_set = @differ.diff(tb_concept, live_concept, detected_at: detected_at)
+
+        merge_localized_content(tb_concept, live_concept)
+
+        unless change_set.empty?
+          add_date(tb_concept, "amended", detected_at)
+          flag_stale_translations(tb_concept, change_set, detected_at)
+        end
+
+        ReconciledConcept.new(
+          managed_concept: tb_concept,
+          change_set: change_set,
+          source: :both,
+        )
+      end
+
+      SOURCE_LANGUAGES = %w[eng fra].freeze
+
+      # When source-language (English/French) notes or definitions change,
+      # the translations in other languages are potentially stale because
+      # they haven't been updated to match. Flag them with an annotation.
+      def flag_stale_translations(concept, change_set, detected_at)
+        source_changes = change_set.select do |c|
+          SOURCE_LANGUAGES.include?(c.language) &&
+            %i[definition notes].include?(c.field)
+        end
+        return if source_changes.empty?
+
+        stale_langs = (concept.localized_concepts&.keys || []) - SOURCE_LANGUAGES
+        return if stale_langs.empty?
+
+        source_fields = source_changes.map { |c| "#{c.language}:#{c.field}" }.uniq.join(", ")
+        stale_langs.each do |lang|
+          lc = concept.localization(lang)
+          next unless lc&.data
+          lc.data.examples ||= []
+          lc.data.examples << Glossarist::DetailedDefinition.new(
+            content: "[Reconciler] Translation may be stale — source language content changed (#{source_fields}) on #{detected_at}",
+          )
+        end
+      end
+
+      def retire(code, tb_concept, detected_at)
+        tb_concept.status = "retired"
+        add_date(tb_concept, "retired", detected_at)
+
+        cs = ChangeSet.new(code)
+        cs.add(Change.new(
+          code: code, field: :status, language: nil,
+          old_value: tb_concept.status, new_value: "retired",
+          detected_at: detected_at,
+        ))
+
+        ReconciledConcept.new(
+          managed_concept: tb_concept,
+          change_set: cs,
+          source: :termbase_only,
+        )
+      end
+
+      def merge_localized_content(merged, live_concept)
+        live_concept.localized_concepts&.each_key do |lang|
+          live_lc = live_concept.localization(lang)
+          next unless live_lc&.data
+
+          existing = merged.localization(lang)
+          if existing&.data
+            update_from_live(existing.data, live_lc.data)
+          else
+            merged.add_l10n(live_lc)
+          end
+        end
+      end
+
+      def update_from_live(target_data, live_data)
+        target_data.definition = live_data.definition if live_data.definition&.any?
+        target_data.notes = live_data.notes if live_data.notes&.any?
+        target_data.examples = live_data.examples if live_data.examples&.any?
+        target_data.terms = live_data.terms if live_data.terms&.any?
+      end
+
+      def add_date(concept, type, date)
+        concept.dates ||= []
+        concept.dates << Glossarist::ConceptDate.new(type: type, date: date)
+      end
+    end
+  end
+end

data/lib/iev/reconciler/content_differ.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # Compares two Glossarist::ManagedConcept objects and produces a
+    # ChangeSet with field-level differences. This powers the change
+    # reporting that answers "what changed, from what to what?"
+    class ContentDiffer
+      # @param old_concept [Glossarist::ManagedConcept] the termbase snapshot
+      # @param new_concept [Glossarist::ManagedConcept] the live snapshot
+      # @param detected_at [String] ISO 8601 date the change was observed
+      # @return [ChangeSet]
+      def diff(old_concept, new_concept, detected_at:)
+        code = new_concept&.id || old_concept&.id
+        change_set = ChangeSet.new(code)
+
+        return change_set unless old_concept && new_concept
+
+        diff_status(change_set, old_concept, new_concept, detected_at)
+        diff_languages(change_set, old_concept, new_concept, detected_at)
+        diff_localized(change_set, old_concept, new_concept, detected_at)
+
+        change_set
+      end
+
+      private
+
+      def diff_status(cs, old_c, new_c, date)
+        old_status = old_c.status
+        new_status = new_c.status
+        return if old_status == new_status
+
+        cs.add(Change.new(
+          code: cs.code,
+          field: :status,
+          language: nil,
+          old_value: old_status,
+          new_value: new_status,
+          detected_at: date,
+        ))
+      end
+
+      def diff_languages(cs, old_c, new_c, date)
+        old_langs = language_keys(old_c)
+        new_langs = language_keys(new_c)
+
+        (new_langs - old_langs).each do |lang|
+          cs.add(Change.new(
+            code: cs.code, field: :language_added, language: lang,
+            old_value: nil, new_value: "present", detected_at: date,
+          ))
+        end
+      end
+
+      def diff_localized(cs, old_c, new_c, date)
+        common_langs = language_keys(old_c) & language_keys(new_c)
+        common_langs.each do |lang|
+          old_lc = localized_data(old_c, lang)
+          new_lc = localized_data(new_c, lang)
+          next unless old_lc && new_lc
+
+          diff_designation(cs, lang, old_lc, new_lc, date)
+          diff_definition(cs, lang, old_lc, new_lc, date)
+          diff_notes(cs, lang, old_lc, new_lc, date)
+          diff_examples(cs, lang, old_lc, new_lc, date)
+        end
+      end
+
+      def diff_designation(cs, lang, old_lc, new_lc, date)
+        old_term = extract_designation(old_lc)
+        new_term = extract_designation(new_lc)
+        return if normalize(old_term) == normalize(new_term)
+
+        cs.add(Change.new(
+          code: cs.code, field: :designation, language: lang,
+          old_value: old_term, new_value: new_term, detected_at: date,
+        ))
+      end
+
+      def diff_definition(cs, lang, old_lc, new_lc, date)
+        old_def = extract_definition(old_lc)
+        new_def = extract_definition(new_lc)
+        return if normalize(old_def) == normalize(new_def)
+
+        cs.add(Change.new(
+          code: cs.code, field: :definition, language: lang,
+          old_value: old_def, new_value: new_def, detected_at: date,
+        ))
+      end
+
+      def diff_notes(cs, lang, old_lc, new_lc, date)
+        old_notes = extract_notes(old_lc)
+        new_notes = extract_notes(new_lc)
+        return if old_notes == new_notes
+
+        cs.add(Change.new(
+          code: cs.code, field: :notes, language: lang,
+          old_value: old_notes.join(" | "), new_value: new_notes.join(" | "),
+          detected_at: date,
+        ))
+      end
+
+      def diff_examples(cs, lang, old_lc, new_lc, date)
+        old_ex = extract_examples(old_lc)
+        new_ex = extract_examples(new_lc)
+        return if old_ex == new_ex
+
+        cs.add(Change.new(
+          code: cs.code, field: :examples, language: lang,
+          old_value: old_ex.join(" | "), new_value: new_ex.join(" | "),
+          detected_at: date,
+        ))
+      end
+
+      # --- extraction helpers ---
+
+      def language_keys(concept)
+        concept.localized_concepts&.keys || []
+      end
+
+      def localized_data(concept, lang)
+        lc = concept.localization(lang)
+        lc&.data
+      end
+
+      def extract_designation(cdata)
+        cdata&.terms&.first&.designation.to_s
+      end
+
+      def extract_definition(cdata)
+        cdata&.definition&.first&.content.to_s
+      end
+
+      def extract_notes(cdata)
+        Array(cdata&.notes).map(&:content)
+      end
+
+      def extract_examples(cdata)
+        Array(cdata&.examples).map(&:content)
+      end
+
+      def normalize(str)
+        return "" unless str
+        str
+          .to_s
+          .unicode_normalize(:nfc)
+          .gsub(/<[^>]+>/, "")
+          .gsub(/[ ​­]/, "")
+          .gsub(/[""″‴„‟‚‛]/, '"')
+          .gsub(/[''‚‛]/, "'")
+          .gsub(/\s+/, " ")
+          .strip
+          .downcase
+      end
+    end
+  end
+end

data/lib/iev/reconciler/live_loader.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Iev
+  module Reconciler
+    # Indexes live HTML pages and builds Glossarist::ManagedConcept
+    # objects on demand per code. Uses Iev::Scraper::PageParser which
+    # runs the full semantic enrichment pipeline (HTML → AsciiDoc with
+    # stem:[] for math, {{urn:...}} for cross-refs, etc.), so parsed
+    # output matches the format used in termbase.yaml.
+    class LiveLoader
+      # @param pages_dir [String, Pathname] directory containing *.html pages
+      def initialize(pages_dir)
+        @pages_dir = pages_dir.to_s
+        @index = nil
+      end
+
+      def codes
+        index.keys
+      end
+
+      def get(code)
+        path = index[code]
+        return nil unless path
+
+        html = File.read(path, encoding: "utf-8")
+        parse_concept(code, html)
+      end
+
+      private
+
+      def index
+        @index ||= Dir.glob(File.join(@pages_dir, "*.html")).each_with_object({}) do |path, h|
+          h[File.basename(path, ".html")] = path
+        end
+      end
+
+      def parse_concept(code, html)
+        doc = Nokogiri::HTML(html)
+        parsed = Iev::Scraper::PageParser.new(doc, code).parse
+        return nil unless parsed && parsed.dig("data", "localized_concepts")&.any?
+
+        build_concept(code, parsed["data"]["localized_concepts"])
+      end
+
+      def build_concept(code, localized_data)
+        concept = Glossarist::ManagedConcept.of_yaml(
+          "id" => code, "data" => { "id" => code },
+        )
+        concept.status = "valid"
+        concept.schema_version = "3"
+
+        localized_data.each do |lang, lc_data|
+          lc = build_localized_concept(code, lang, lc_data)
+          concept.add_l10n(lc)
+        end
+
+        concept
+      end
+
+      def build_localized_concept(code, lang, lc_data)
+        cdata = Glossarist::ConceptData.new
+        cdata.id = code
+        cdata.language_code = lang
+        cdata.entry_status = "valid"
+
+        term = lc_data["term"].to_s
+        areas = lc_data["term_areas"] || {}
+        cdata.terms = if term.empty?
+                        []
+                      else
+                        TermMarkerParser.parse_multiple(term).map do |parsed|
+                          build_expression(parsed, areas[parsed.designation])
+                        end
+                      end
+
+        definition = lc_data["definition"]
+        if definition && !definition.empty?
+          parts = split_notes_examples(definition)
+          if parts[:definition] && !parts[:definition].empty?
+            cdata.definition = [Glossarist::DetailedDefinition.new(content: parts[:definition])]
+          end
+          cdata.notes = parts[:notes].map { |n| Glossarist::DetailedDefinition.new(content: n) }
+          cdata.examples = parts[:examples].map { |e| Glossarist::DetailedDefinition.new(content: e) }
+        end
+
+        lc = Glossarist::LocalizedConcept.new
+        lc.id = code
+        lc.data = cdata
+        lc
+      end
+
+      def build_expression(parsed, area = nil)
+        expr = Glossarist::Designation::Expression.new(
+          designation: parsed.designation,
+          normative_status: "preferred",
+        )
+        expr.geographical_area = area if area
+        expr.field_of_application = parsed.domain if parsed.domain
+        expr.usage_info = parsed.usage_info if parsed.usage_info
+        expr.prefix = parsed.is_prefix if parsed.is_prefix
+
+        grammar = build_grammar_info(parsed)
+        expr.grammar_info = [grammar] if grammar
+        expr
+      end
+
+      def build_related_concepts(parsed)
+        return [] unless parsed.related_refs&.any?
+
+        parsed.related_refs.map do |code|
+          Glossarist::RelatedConcept.new(
+            type: "see",
+            ref: { "source" => "IEV", "id" => code },
+          )
+        end
+      end
+
+      def build_grammar_info(parsed)
+        has_gender = parsed.genders&.any?
+        has_number = parsed.numbers&.any?
+        has_pos = !parsed.part_of_speech.nil?
+        return nil unless has_gender || has_number || has_pos
+
+        grammar = Glossarist::Designation::GrammarInfo.new
+        grammar.gender = parsed.genders if has_gender
+        grammar.number = parsed.numbers if has_number
+        grammar.part_of_speech = parsed.part_of_speech if has_pos
+        grammar
+      end
+
+      NOTE_RE = /^(Note\s+\d+\s+to\s+entry:.*)$/i
+      EXAMPLE_RE = /^(Example\s*:.*)$/i
+      NOTE_PREFIX_RE = /^Note\s+\d+\s+to\s+entry:\s*/i
+      EXAMPLE_PREFIX_RE = /^Example\s*:\s*/i
+
+      def split_notes_examples(text)
+        notes = []
+        examples = []
+
+        lines = text.split("\n")
+        definition_lines = []
+
+        lines.each do |line|
+          stripped = line.strip
+          if stripped.match?(NOTE_RE)
+            notes << stripped.sub(NOTE_PREFIX_RE, "")
+          elsif stripped.match?(EXAMPLE_RE)
+            examples << stripped.sub(EXAMPLE_PREFIX_RE, "")
+          else
+            definition_lines << line
+          end
+        end
+
+        definition = definition_lines.join("\n").strip
+        definition = nil if definition.empty?
+
+        { definition: definition, notes: notes, examples: examples }
+      end
+    end
+  end
+end