iev 0.4.6 → 0.4.8

data/lib/iev/reconciler/content_differ.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # Compares two Glossarist::ManagedConcept objects and produces a
+    # ChangeSet with field-level differences. This powers the change
+    # reporting that answers "what changed, from what to what?"
+    class ContentDiffer
+      # @param old_concept [Glossarist::ManagedConcept] the termbase snapshot
+      # @param new_concept [Glossarist::ManagedConcept] the live snapshot
+      # @param detected_at [String] ISO 8601 date the change was observed
+      # @return [ChangeSet]
+      def diff(old_concept, new_concept, detected_at:)
+        code = new_concept&.id || old_concept&.id
+        change_set = ChangeSet.new(code)
+
+        return change_set unless old_concept && new_concept
+
+        diff_status(change_set, old_concept, new_concept, detected_at)
+        diff_languages(change_set, old_concept, new_concept, detected_at)
+        diff_localized(change_set, old_concept, new_concept, detected_at)
+
+        change_set
+      end
+
+      private
+
+      def diff_status(cs, old_c, new_c, date)
+        old_status = old_c.status
+        new_status = new_c.status
+        return if old_status == new_status
+
+        cs.add(Change.new(
+          code: cs.code,
+          field: :status,
+          language: nil,
+          old_value: old_status,
+          new_value: new_status,
+          detected_at: date,
+        ))
+      end
+
+      def diff_languages(cs, old_c, new_c, date)
+        old_langs = language_keys(old_c)
+        new_langs = language_keys(new_c)
+
+        (new_langs - old_langs).each do |lang|
+          cs.add(Change.new(
+            code: cs.code, field: :language_added, language: lang,
+            old_value: nil, new_value: "present", detected_at: date,
+          ))
+        end
+      end
+
+      def diff_localized(cs, old_c, new_c, date)
+        common_langs = language_keys(old_c) & language_keys(new_c)
+        common_langs.each do |lang|
+          old_lc = localized_data(old_c, lang)
+          new_lc = localized_data(new_c, lang)
+          next unless old_lc && new_lc
+
+          diff_designation(cs, lang, old_lc, new_lc, date)
+          diff_definition(cs, lang, old_lc, new_lc, date)
+          diff_notes(cs, lang, old_lc, new_lc, date)
+          diff_examples(cs, lang, old_lc, new_lc, date)
+        end
+      end
+
+      def diff_designation(cs, lang, old_lc, new_lc, date)
+        old_term = extract_designation(old_lc)
+        new_term = extract_designation(new_lc)
+        return if normalize(old_term) == normalize(new_term)
+
+        cs.add(Change.new(
+          code: cs.code, field: :designation, language: lang,
+          old_value: old_term, new_value: new_term, detected_at: date,
+        ))
+      end
+
+      def diff_definition(cs, lang, old_lc, new_lc, date)
+        old_def = extract_definition(old_lc)
+        new_def = extract_definition(new_lc)
+        return if normalize(old_def) == normalize(new_def)
+
+        cs.add(Change.new(
+          code: cs.code, field: :definition, language: lang,
+          old_value: old_def, new_value: new_def, detected_at: date,
+        ))
+      end
+
+      def diff_notes(cs, lang, old_lc, new_lc, date)
+        old_notes = extract_notes(old_lc)
+        new_notes = extract_notes(new_lc)
+        return if old_notes == new_notes
+
+        cs.add(Change.new(
+          code: cs.code, field: :notes, language: lang,
+          old_value: old_notes.join(" | "), new_value: new_notes.join(" | "),
+          detected_at: date,
+        ))
+      end
+
+      def diff_examples(cs, lang, old_lc, new_lc, date)
+        old_ex = extract_examples(old_lc)
+        new_ex = extract_examples(new_lc)
+        return if old_ex == new_ex
+
+        cs.add(Change.new(
+          code: cs.code, field: :examples, language: lang,
+          old_value: old_ex.join(" | "), new_value: new_ex.join(" | "),
+          detected_at: date,
+        ))
+      end
+
+      # --- extraction helpers ---
+
+      def language_keys(concept)
+        concept.localized_concepts&.keys || []
+      end
+
+      def localized_data(concept, lang)
+        lc = concept.localization(lang)
+        lc&.data
+      end
+
+      def extract_designation(cdata)
+        cdata&.terms&.first&.designation.to_s
+      end
+
+      def extract_definition(cdata)
+        cdata&.definition&.first&.content.to_s
+      end
+
+      def extract_notes(cdata)
+        Array(cdata&.notes).map(&:content)
+      end
+
+      def extract_examples(cdata)
+        Array(cdata&.examples).map(&:content)
+      end
+
+      def normalize(str)
+        return "" unless str
+        str
+          .to_s
+          .unicode_normalize(:nfc)
+          .gsub(/<[^>]+>/, "")
+          .gsub(/[ ​­]/, "")
+          .gsub(/[""″‴„‟‚‛]/, '"')
+          .gsub(/[''‚‛]/, "'")
+          .gsub(/\s+/, " ")
+          .strip
+          .downcase
+      end
+    end
+  end
+end

data/lib/iev/reconciler/live_loader.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Iev
+  module Reconciler
+    # Indexes live HTML pages and builds Glossarist::ManagedConcept
+    # objects on demand per code. Uses Iev::Scraper::PageParser which
+    # runs the full semantic enrichment pipeline (HTML → AsciiDoc with
+    # stem:[] for math, {{urn:...}} for cross-refs, etc.), so parsed
+    # output matches the format used in termbase.yaml.
+    class LiveLoader
+      # @param pages_dir [String, Pathname] directory containing *.html pages
+      def initialize(pages_dir)
+        @pages_dir = pages_dir.to_s
+        @index = nil
+      end
+
+      def codes
+        index.keys
+      end
+
+      def get(code)
+        path = index[code]
+        return nil unless path
+
+        html = File.read(path, encoding: "utf-8")
+        parse_concept(code, html)
+      end
+
+      private
+
+      def index
+        @index ||= Dir.glob(File.join(@pages_dir, "*.html")).each_with_object({}) do |path, h|
+          h[File.basename(path, ".html")] = path
+        end
+      end
+
+      def parse_concept(code, html)
+        doc = Nokogiri::HTML(html)
+        parsed = Iev::Scraper::PageParser.new(doc, code).parse
+        return nil unless parsed && parsed.dig("data", "localized_concepts")&.any?
+
+        build_concept(code, parsed["data"]["localized_concepts"])
+      end
+
+      def build_concept(code, localized_data)
+        concept = Glossarist::ManagedConcept.of_yaml(
+          "id" => code, "data" => { "id" => code },
+        )
+        concept.status = "valid"
+        concept.schema_version = "3"
+
+        localized_data.each do |lang, lc_data|
+          lc = build_localized_concept(code, lang, lc_data)
+          concept.add_l10n(lc)
+        end
+
+        concept
+      end
+
+      def build_localized_concept(code, lang, lc_data)
+        cdata = Glossarist::ConceptData.new
+        cdata.id = code
+        cdata.language_code = lang
+        cdata.entry_status = "valid"
+
+        term = lc_data["term"].to_s
+        areas = lc_data["term_areas"] || {}
+        cdata.terms = if term.empty?
+                        []
+                      else
+                        TermMarkerParser.parse_multiple(term).map do |parsed|
+                          build_expression(parsed, areas[parsed.designation])
+                        end
+                      end
+
+        definition = lc_data["definition"]
+        if definition && !definition.empty?
+          parts = split_notes_examples(definition)
+          if parts[:definition] && !parts[:definition].empty?
+            cdata.definition = [Glossarist::DetailedDefinition.new(content: parts[:definition])]
+          end
+          cdata.notes = parts[:notes].map { |n| Glossarist::DetailedDefinition.new(content: n) }
+          cdata.examples = parts[:examples].map { |e| Glossarist::DetailedDefinition.new(content: e) }
+        end
+
+        lc = Glossarist::LocalizedConcept.new
+        lc.id = code
+        lc.data = cdata
+        lc
+      end
+
+      def build_expression(parsed, area = nil)
+        expr = Glossarist::Designation::Expression.new(
+          designation: parsed.designation,
+          normative_status: "preferred",
+        )
+        expr.geographical_area = area if area
+        expr.field_of_application = parsed.domain if parsed.domain
+        expr.usage_info = parsed.usage_info if parsed.usage_info
+        expr.prefix = parsed.is_prefix if parsed.is_prefix
+
+        grammar = build_grammar_info(parsed)
+        expr.grammar_info = [grammar] if grammar
+        expr
+      end
+
+      def build_related_concepts(parsed)
+        return [] unless parsed.related_refs&.any?
+
+        parsed.related_refs.map do |code|
+          Glossarist::RelatedConcept.new(
+            type: "see",
+            ref: { "source" => "IEV", "id" => code },
+          )
+        end
+      end
+
+      def build_grammar_info(parsed)
+        has_gender = parsed.genders&.any?
+        has_number = parsed.numbers&.any?
+        has_pos = !parsed.part_of_speech.nil?
+        return nil unless has_gender || has_number || has_pos
+
+        grammar = Glossarist::Designation::GrammarInfo.new
+        grammar.gender = parsed.genders if has_gender
+        grammar.number = parsed.numbers if has_number
+        grammar.part_of_speech = parsed.part_of_speech if has_pos
+        grammar
+      end
+
+      NOTE_RE = /^(Note\s+\d+\s+to\s+entry:.*)$/i
+      EXAMPLE_RE = /^(Example\s*:.*)$/i
+      NOTE_PREFIX_RE = /^Note\s+\d+\s+to\s+entry:\s*/i
+      EXAMPLE_PREFIX_RE = /^Example\s*:\s*/i
+
+      def split_notes_examples(text)
+        notes = []
+        examples = []
+
+        lines = text.split("\n")
+        definition_lines = []
+
+        lines.each do |line|
+          stripped = line.strip
+          if stripped.match?(NOTE_RE)
+            notes << stripped.sub(NOTE_PREFIX_RE, "")
+          elsif stripped.match?(EXAMPLE_RE)
+            examples << stripped.sub(EXAMPLE_PREFIX_RE, "")
+          else
+            definition_lines << line
+          end
+        end
+
+        definition = definition_lines.join("\n").strip
+        definition = nil if definition.empty?
+
+        { definition: definition, notes: notes, examples: examples }
+      end
+    end
+  end
+end

data/lib/iev/reconciler/pipeline.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "date"
+require "fileutils"
+require "yaml"
+
+module Iev
+  module Reconciler
+    # Orchestrates the full reconciliation by streaming through codes
+    # one at a time. Each code is loaded → merged → serialized → discarded,
+    # so memory usage is bounded regardless of dataset size.
+    class Pipeline
+      attr_reader :stats
+
+      # @param termbase_path [String] path to termbase.yaml
+      # @param pages_dir [String] directory with mirrored HTML pages
+      # @param output_dir [String] where to write concepts + reports
+      # @param detected_at [String] ISO 8601 date for detected changes
+      def initialize(termbase_path:, pages_dir:, output_dir:,
+                     detected_at: Date.today.iso8601)
+        @termbase_path = termbase_path
+        @pages_dir = pages_dir
+        @output_dir = output_dir
+        @detected_at = detected_at
+        @stats = {}
+      end
+
+      # Run the full pipeline.
+      # @return [void]
+      def run
+        FileUtils.mkdir_p(File.join(@output_dir, "concepts"))
+        FileUtils.mkdir_p(File.join(@output_dir, "report"))
+
+        warn "Indexing sources..."
+        termbase = TermbaseLoader.new(@termbase_path)
+        live = LiveLoader.new(@pages_dir)
+        all_codes = (termbase.codes + live.codes).uniq.sort
+        warn "  #{termbase.codes.size} termbase + #{live.codes.size} live = #{all_codes.size} total"
+
+        merger = ConceptMerger.new
+        reconciled = []
+        errors = []
+
+        all_codes.each_with_index do |code, idx|
+          begin
+            rc = merger.merge(
+              code: code,
+              termbase_concept: termbase.get(code),
+              live_concept: live.get(code),
+              detected_at: @detected_at,
+            )
+
+            if rc
+              reconciled << rc
+              save_concept(rc.managed_concept)
+            end
+          rescue StandardError => e
+            errors << { code: code, error: "#{e.class}: #{e.message}" }
+            warn "  ERROR on #{code}: #{e.message[0, 100]}"
+          end
+
+          if (idx + 1) % 1000 == 0
+            warn "  #{idx + 1}/#{all_codes.size}... (#{errors.size} errors)"
+          end
+        end
+
+        compute_stats(reconciled)
+        compute_error_stats(errors) if errors.any?
+        Report.new(reconciled).write_to(File.join(@output_dir, "report"))
+        warn "Done."
+      end
+
+      private
+
+      def save_concept(concept)
+        path = File.join(@output_dir, "concepts", "#{concept.id}.yaml")
+        Iev::MultiDocYaml.write(path, Iev::MultiDocYaml.parts_for(concept))
+      end
+
+      def compute_stats(reconciled)
+        @stats = {
+          total: reconciled.size,
+          in_both: reconciled.count { |r| r.source == :both },
+          termbase_only: reconciled.count { |r| r.source == :termbase_only },
+          live_only: reconciled.count { |r| r.source == :live_only },
+          changed: reconciled.count { |r| !r.change_set.empty? },
+        }
+        warn ""
+        warn "Results:"
+        warn "  In both (merged):        #{@stats[:in_both]}"
+        warn "  Termbase only (retired): #{@stats[:termbase_only]}"
+        warn "  Live only (new):         #{@stats[:live_only]}"
+        warn "  Concepts with changes:   #{@stats[:changed]}"
+        warn "  Total:                   #{@stats[:total]}"
+      end
+
+      def compute_error_stats(errors)
+        warn ""
+        warn "Errors: #{errors.size}"
+        errors.first(10).each { |e| warn "  #{e[:code]}: #{e[:error]}" }
+      end
+    end
+  end
+end

data/lib/iev/reconciler/reconciled_concept.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # Wraps a reconciled ManagedConcept with its ChangeSet and origin.
+    # Value object produced by ConceptMerger, consumed by Pipeline + Report.
+    #
+    # @attr managed_concept [Glossarist::ManagedConcept]
+    # @attr change_set [ChangeSet] field-level diffs detected
+    # @attr source [Symbol] :both | :termbase_only | :live_only
+    ReconciledConcept = Struct.new(:managed_concept, :change_set, :source,
+                                   keyword_init: true)
+  end
+end

data/lib/iev/reconciler/report.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "csv"
+require "yaml"
+
+module Iev
+  module Reconciler
+    # Generates dataset-level change reports from reconciled concepts.
+    # Answers "what changed across the entire IEV?" with machine-readable
+    # summary files.
+    class Report
+      # @param reconciled [Array<ReconciledConcept>] all reconciled concepts
+      def initialize(reconciled)
+        @reconciled = reconciled
+      end
+
+      # Write all report files to the given directory.
+      # @param dir [String, Pathname]
+      def write_to(dir)
+        require "fileutils"
+        FileUtils.mkdir_p(dir)
+
+        write_summary(File.join(dir, "summary.yaml"))
+        write_changes_csv(File.join(dir, "changes.csv"))
+        write_retired(File.join(dir, "retired.yaml"))
+        write_new_concepts(File.join(dir, "new_concepts.yaml"))
+      end
+
+      # @return [Hash] aggregate statistics
+      def summary
+        {
+          total_concepts: @reconciled.size,
+          sources: source_counts,
+          changes: change_stats,
+        }
+      end
+
+      private
+
+      def write_summary(path)
+        File.write(path, YAML.dump(summary))
+      end
+
+      def write_changes_csv(path)
+        CSV.open(path, "w") do |csv|
+          csv << %w[code section field language detected_at old_value new_value]
+          @reconciled.each do |rc|
+            rc.change_set.each do |change|
+              csv << [
+                change.code,
+                section_of(change.code),
+                change.field,
+                change.language,
+                change.detected_at,
+                truncate(change.old_value),
+                truncate(change.new_value),
+              ]
+            end
+          end
+        end
+      end
+
+      def write_retired(path)
+        retired = @reconciled
+          .select { |rc| rc.source == :termbase_only && rc.managed_concept }
+          .map { |rc| rc.managed_concept.id }
+        File.write(path, YAML.dump(retired))
+      end
+
+      def write_new_concepts(path)
+        new_concepts = @reconciled
+          .select { |rc| rc.source == :live_only && rc.managed_concept }
+          .map { |rc| rc.managed_concept.id }
+        File.write(path, YAML.dump(new_concepts))
+      end
+
+      def source_counts
+        @reconciled.group_by(&:source).transform_values(&:size)
+      end
+
+      def change_stats
+        all_changes = @reconciled.flat_map { |rc| rc.change_set.to_a }
+        {
+          total: all_changes.size,
+          by_field: tally_by(all_changes, :field),
+          by_language: tally_by(all_changes, :language),
+          by_section: tally_by_section(all_changes),
+        }
+      end
+
+      def tally_by(changes, attr)
+        changes
+          .group_by { |c| c.send(attr) }
+          .transform_values(&:size)
+          .sort_by { |_, v| -v }
+          .to_h
+      end
+
+      def tally_by_section(changes)
+        changes
+          .group_by { |c| section_of(c.code) }
+          .transform_values(&:size)
+          .sort_by { |_, v| -v }
+          .to_h
+      end
+
+      def section_of(code)
+        code.to_s.rpartition("-").first
+      end
+
+      def truncate(value, max = 200)
+        return "" if value.nil?
+        value = value.to_s
+        value.size > max ? value[0, max] + "..." : value
+      end
+    end
+  end
+end

data/lib/iev/reconciler/status_mapper.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # Maps termbase entry_status strings to Glossarist V3 ConceptStatus
+    # enum values. Stateless and pure.
+    module StatusMapper
+      MAP = {
+        "Standard" => "valid",
+        "Published" => "valid",
+        "Effective" => "valid",
+        "Draft" => "draft",
+        "Not Valid" => "notValid",
+        "Superseded" => "superseded",
+        "Retired" => "retired",
+      }.freeze
+
+      DEFAULT = "valid"
+
+      module_function
+
+      # @param entry_status [String, nil]
+      # @return [String] a valid ConceptStatus value
+      def call(entry_status)
+        MAP[entry_status.to_s] || DEFAULT
+      end
+    end
+  end
+end