iev 0.4.7 → 0.4.8

data/lib/iev/reconciler/pipeline.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "date"
+require "fileutils"
+require "yaml"
+
+module Iev
+  module Reconciler
+    # Orchestrates the full reconciliation by streaming through codes
+    # one at a time. Each code is loaded → merged → serialized → discarded,
+    # so memory usage is bounded regardless of dataset size.
+    class Pipeline
+      attr_reader :stats
+
+      # @param termbase_path [String] path to termbase.yaml
+      # @param pages_dir [String] directory with mirrored HTML pages
+      # @param output_dir [String] where to write concepts + reports
+      # @param detected_at [String] ISO 8601 date for detected changes
+      def initialize(termbase_path:, pages_dir:, output_dir:,
+                     detected_at: Date.today.iso8601)
+        @termbase_path = termbase_path
+        @pages_dir = pages_dir
+        @output_dir = output_dir
+        @detected_at = detected_at
+        @stats = {}
+      end
+
+      # Run the full pipeline.
+      # @return [void]
+      def run
+        FileUtils.mkdir_p(File.join(@output_dir, "concepts"))
+        FileUtils.mkdir_p(File.join(@output_dir, "report"))
+
+        warn "Indexing sources..."
+        termbase = TermbaseLoader.new(@termbase_path)
+        live = LiveLoader.new(@pages_dir)
+        all_codes = (termbase.codes + live.codes).uniq.sort
+        warn "  #{termbase.codes.size} termbase + #{live.codes.size} live = #{all_codes.size} total"
+
+        merger = ConceptMerger.new
+        reconciled = []
+        errors = []
+
+        all_codes.each_with_index do |code, idx|
+          begin
+            rc = merger.merge(
+              code: code,
+              termbase_concept: termbase.get(code),
+              live_concept: live.get(code),
+              detected_at: @detected_at,
+            )
+
+            if rc
+              reconciled << rc
+              save_concept(rc.managed_concept)
+            end
+          rescue StandardError => e
+            errors << { code: code, error: "#{e.class}: #{e.message}" }
+            warn "  ERROR on #{code}: #{e.message[0, 100]}"
+          end
+
+          if (idx + 1) % 1000 == 0
+            warn "  #{idx + 1}/#{all_codes.size}... (#{errors.size} errors)"
+          end
+        end
+
+        compute_stats(reconciled)
+        compute_error_stats(errors) if errors.any?
+        Report.new(reconciled).write_to(File.join(@output_dir, "report"))
+        warn "Done."
+      end
+
+      private
+
+      def save_concept(concept)
+        path = File.join(@output_dir, "concepts", "#{concept.id}.yaml")
+        Iev::MultiDocYaml.write(path, Iev::MultiDocYaml.parts_for(concept))
+      end
+
+      def compute_stats(reconciled)
+        @stats = {
+          total: reconciled.size,
+          in_both: reconciled.count { |r| r.source == :both },
+          termbase_only: reconciled.count { |r| r.source == :termbase_only },
+          live_only: reconciled.count { |r| r.source == :live_only },
+          changed: reconciled.count { |r| !r.change_set.empty? },
+        }
+        warn ""
+        warn "Results:"
+        warn "  In both (merged):        #{@stats[:in_both]}"
+        warn "  Termbase only (retired): #{@stats[:termbase_only]}"
+        warn "  Live only (new):         #{@stats[:live_only]}"
+        warn "  Concepts with changes:   #{@stats[:changed]}"
+        warn "  Total:                   #{@stats[:total]}"
+      end
+
+      def compute_error_stats(errors)
+        warn ""
+        warn "Errors: #{errors.size}"
+        errors.first(10).each { |e| warn "  #{e[:code]}: #{e[:error]}" }
+      end
+    end
+  end
+end

data/lib/iev/reconciler/reconciled_concept.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # Wraps a reconciled ManagedConcept with its ChangeSet and origin.
+    # Value object produced by ConceptMerger, consumed by Pipeline + Report.
+    #
+    # @attr managed_concept [Glossarist::ManagedConcept]
+    # @attr change_set [ChangeSet] field-level diffs detected
+    # @attr source [Symbol] :both | :termbase_only | :live_only
+    ReconciledConcept = Struct.new(:managed_concept, :change_set, :source,
+                                   keyword_init: true)
+  end
+end

data/lib/iev/reconciler/report.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "csv"
+require "yaml"
+
+module Iev
+  module Reconciler
+    # Generates dataset-level change reports from reconciled concepts.
+    # Answers "what changed across the entire IEV?" with machine-readable
+    # summary files.
+    class Report
+      # @param reconciled [Array<ReconciledConcept>] all reconciled concepts
+      def initialize(reconciled)
+        @reconciled = reconciled
+      end
+
+      # Write all report files to the given directory.
+      # @param dir [String, Pathname]
+      def write_to(dir)
+        require "fileutils"
+        FileUtils.mkdir_p(dir)
+
+        write_summary(File.join(dir, "summary.yaml"))
+        write_changes_csv(File.join(dir, "changes.csv"))
+        write_retired(File.join(dir, "retired.yaml"))
+        write_new_concepts(File.join(dir, "new_concepts.yaml"))
+      end
+
+      # @return [Hash] aggregate statistics
+      def summary
+        {
+          total_concepts: @reconciled.size,
+          sources: source_counts,
+          changes: change_stats,
+        }
+      end
+
+      private
+
+      def write_summary(path)
+        File.write(path, YAML.dump(summary))
+      end
+
+      def write_changes_csv(path)
+        CSV.open(path, "w") do |csv|
+          csv << %w[code section field language detected_at old_value new_value]
+          @reconciled.each do |rc|
+            rc.change_set.each do |change|
+              csv << [
+                change.code,
+                section_of(change.code),
+                change.field,
+                change.language,
+                change.detected_at,
+                truncate(change.old_value),
+                truncate(change.new_value),
+              ]
+            end
+          end
+        end
+      end
+
+      def write_retired(path)
+        retired = @reconciled
+          .select { |rc| rc.source == :termbase_only && rc.managed_concept }
+          .map { |rc| rc.managed_concept.id }
+        File.write(path, YAML.dump(retired))
+      end
+
+      def write_new_concepts(path)
+        new_concepts = @reconciled
+          .select { |rc| rc.source == :live_only && rc.managed_concept }
+          .map { |rc| rc.managed_concept.id }
+        File.write(path, YAML.dump(new_concepts))
+      end
+
+      def source_counts
+        @reconciled.group_by(&:source).transform_values(&:size)
+      end
+
+      def change_stats
+        all_changes = @reconciled.flat_map { |rc| rc.change_set.to_a }
+        {
+          total: all_changes.size,
+          by_field: tally_by(all_changes, :field),
+          by_language: tally_by(all_changes, :language),
+          by_section: tally_by_section(all_changes),
+        }
+      end
+
+      def tally_by(changes, attr)
+        changes
+          .group_by { |c| c.send(attr) }
+          .transform_values(&:size)
+          .sort_by { |_, v| -v }
+          .to_h
+      end
+
+      def tally_by_section(changes)
+        changes
+          .group_by { |c| section_of(c.code) }
+          .transform_values(&:size)
+          .sort_by { |_, v| -v }
+          .to_h
+      end
+
+      def section_of(code)
+        code.to_s.rpartition("-").first
+      end
+
+      def truncate(value, max = 200)
+        return "" if value.nil?
+        value = value.to_s
+        value.size > max ? value[0, max] + "..." : value
+      end
+    end
+  end
+end

data/lib/iev/reconciler/status_mapper.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # Maps termbase entry_status strings to Glossarist V3 ConceptStatus
+    # enum values. Stateless and pure.
+    module StatusMapper
+      MAP = {
+        "Standard" => "valid",
+        "Published" => "valid",
+        "Effective" => "valid",
+        "Draft" => "draft",
+        "Not Valid" => "notValid",
+        "Superseded" => "superseded",
+        "Retired" => "retired",
+      }.freeze
+
+      DEFAULT = "valid"
+
+      module_function
+
+      # @param entry_status [String, nil]
+      # @return [String] a valid ConceptStatus value
+      def call(entry_status)
+        MAP[entry_status.to_s] || DEFAULT
+      end
+    end
+  end
+end

data/lib/iev/reconciler/term_marker_parser.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    # Extracts grammatical markers, context qualifiers, and part-of-speech
+    # tags that are inline in Electropedia's live HTML term text, matching
+    # how the termbase stores them in separate fields.
+    #
+    # Two-pass approach:
+    #   Pass 1: extract context qualifiers and IEV cross-references from
+    #           angle brackets (<text>, <相关条目：IEV xxx>)
+    #   Pass 2: extract gender/number/POS/prefix markers from remaining text
+    #
+    # Marker formats handled:
+    #   Gender:      , f  , m  , n  (also multiple: , f, n)
+    #                , ж  , м  , с  (Serbian Cyrillic)
+    #                , m/f  (slash = both apply)
+    #   Number:      , pl , sg  (also Serbian: јд, мн)
+    #                , f pl  (space-separated gender+number)
+    #   Usage info:  <text> → usage_info field
+    #   Part of speech: , 名詞 , 명사 , noun , verb , adj. , agg , etc.
+    #   Prefix:      , Präfix , 접두사 , 接頭語 , etc. → prefix flag
+    class TermMarkerParser
+      GENDER_MAP = {
+        "f" => "feminine", "m" => "masculine", "n" => "neuter",
+        "ж" => "feminine", "м" => "masculine", "с" => "neuter",
+      }.freeze
+
+      NUMBER_MAP = {
+        "јд" => "singular", "мн" => "plural",
+        "sg" => "singular", "pl" => "plural",
+      }.freeze
+
+      PART_OF_SPEECH_MAP = {
+        "noun" => "noun",
+        "명사" => "noun", "名詞" => "noun", "اسم" => "noun",
+        "именица" => "noun",
+        "verb" => "verb", "verbo" => "verb",
+        "동사" => "verb", "動詞" => "verb", "فعل" => "verb",
+        "глагол" => "verb",
+        "adj" => "adj", "adj." => "adj", "adjective" => "adj",
+        "agg" => "adj", "agg." => "adj",
+        "형용사" => "adj", "形容詞" => "adj", "صفة" => "adj",
+        "adjektiv" => "adj", "придев" => "adj",
+        "adv" => "adv", "adv." => "adv", "adverb" => "adv",
+        "부사" => "adv", "副詞" => "adv",
+        "прилог" => "adv",
+      }.freeze
+
+      PREFIX_KEYWORDS = %w[
+        Präfix prefix préfixe 접두사 接頭語 接尾語
+      ].freeze
+
+      DOMAIN_RE = /<([^>]+)>/
+      USAGE_INFO_RE = /\(([^)]+)\)\s*(?=[,]?\s*(?:[fmnжмс]\b|[fmn]\/[fmn]|sg|pl|јд|мн|\z))/i
+      IEV_XREF_RE = /<[^>]*IEV\s*(\d{3}-\d{2}-\d{2,3})[^>]*>/i
+
+      SERBIAN_RE = /,\s*([жмс])\s+(јд|мн)\s*\z/
+      WESTERN_GENDER_NUMBER_RE = /,\s*([fmn])\s+(sg|pl)\.?\s*\z/i
+      WESTERN_NUMBER_RE = /,\s*(sg|pl)\.?\s*\z/i
+      SLASH_GENDER_RE = /,\s*([fmn])\/([fmn])\s*\z/i
+      WESTERN_RE = /[,]?\s+([fmn])\s*\z/
+      SPACE_GENDER_RE = /\s+([fmn])\s*\z/
+
+      POS_RE = /,\s*(#{PART_OF_SPEECH_MAP.keys.map { |k| Regexp.escape(k) }.join("|")})\s*\z/i
+      PREFIX_RE = /,\s*(#{PREFIX_KEYWORDS.map { |k| Regexp.escape(k) }.join("|")})\s*\z/
+
+      TRAILING_PUNCT_RE = /[,;\s]+$/
+
+      Result = Struct.new(:designation, :genders, :numbers, :domain,
+                          :usage_info, :part_of_speech, :related_refs,
+                          :is_prefix, keyword_init: true)
+
+      class << self
+        def parse(term)
+          return Result.new(designation: nil) unless term
+
+          text = decode_entities(term).strip.gsub(/\s+/, " ").gsub(/　/, " ")
+          related_refs, text = extract_iev_xrefs(text)
+          domain, text = extract_domain(text)
+          usage_info, text = extract_usage_info(text)
+          designation, genders, numbers, pos, is_prefix = extract_markers(text)
+
+          Result.new(
+            designation: designation,
+            genders: genders,
+            numbers: numbers,
+            domain: domain,
+            usage_info: usage_info,
+            part_of_speech: pos,
+            related_refs: related_refs,
+            is_prefix: is_prefix,
+          )
+        end
+
+        def parse_multiple(term_text)
+          return [] unless term_text
+
+          term_text
+            .split(/\n+/)
+            .map(&:strip)
+            .reject(&:empty?)
+            .map { |part| parse(part) }
+        end
+
+        private
+
+        def decode_entities(str)
+          str
+            .gsub(/&lt;/, "<")
+            .gsub(/&gt;/, ">")
+            .gsub(/&amp;/, "&")
+            .gsub(/&quot;/, '"')
+            .gsub(/&#39;/, "'")
+        end
+
+        # Extract domain qualifier from angle brackets: <text> -> domain
+        def extract_domain(text)
+          match = text.match(DOMAIN_RE)
+          return [nil, text] unless match
+
+          domain = match[1].strip
+          remaining = text.sub(match[0], "").gsub(TRAILING_PUNCT_RE, "").strip
+          remaining = remaining.sub(/^[,;\s]+/, "").strip
+          [domain, remaining]
+        end
+
+        def extract_iev_xrefs(text)
+          refs = []
+          remaining = text
+          while (match = remaining.match(IEV_XREF_RE))
+            refs << match[1]
+            remaining = remaining.sub(match[0], "").gsub(TRAILING_PUNCT_RE, "").strip
+            remaining = remaining.sub(/^[,;\s]+/, "").strip
+          end
+          [refs.empty? ? nil : refs, remaining]
+        end
+
+        # Extract usage_info from parentheses that precede trailing markers.
+        # E.g. "Orientierung (einer Kurve) f" -> usage_info="einer Kurve"
+        def extract_usage_info(text)
+          match = text.match(USAGE_INFO_RE)
+          return [nil, text] unless match
+
+          usage = match[1].strip
+          remaining = text.sub(match[0], "").gsub(/\s+/, " ").strip
+          [usage, remaining]
+        end
+
+        def extract_markers(text)
+          designation = text
+          genders = []
+          numbers = []
+          pos = nil
+          is_prefix = false
+
+          loop do
+            break if designation.nil? || designation.empty?
+
+            if (m = designation.match(SERBIAN_RE))
+              designation = designation[0, m.begin(0)].strip
+              genders << GENDER_MAP[m[1]] if GENDER_MAP[m[1]]
+              numbers << NUMBER_MAP[m[2]] if NUMBER_MAP[m[2]]
+            elsif (m = designation.match(SLASH_GENDER_RE))
+              designation = designation[0, m.begin(0)].strip
+              genders << GENDER_MAP[m[1]] if GENDER_MAP[m[1]]
+              genders << GENDER_MAP[m[2]] if GENDER_MAP[m[2]]
+            elsif (m = designation.match(WESTERN_GENDER_NUMBER_RE))
+              designation = designation[0, m.begin(0)].strip
+              genders << GENDER_MAP[m[1]] if GENDER_MAP[m[1]]
+              num_key = m[2].downcase.sub(/\.$/, "")
+              numbers << NUMBER_MAP[num_key] if NUMBER_MAP[num_key]
+            elsif (m = designation.match(WESTERN_NUMBER_RE))
+              designation = designation[0, m.begin(0)].strip
+              num_key = m[1].downcase.sub(/\.$/, "")
+              numbers << NUMBER_MAP[num_key] if NUMBER_MAP[num_key]
+            elsif (m = designation.match(POS_RE))
+              designation = designation[0, m.begin(0)].strip
+              pos = PART_OF_SPEECH_MAP[m[1].downcase]
+            elsif (m = designation.match(PREFIX_RE))
+              designation = designation[0, m.begin(0)].strip
+              is_prefix = true
+            elsif (m = designation.match(WESTERN_RE))
+              designation = designation[0, m.begin(0)].strip
+              genders << GENDER_MAP[m[1]] if GENDER_MAP[m[1]]
+            else
+              break
+            end
+          end
+
+          [designation, genders.uniq, numbers.uniq, pos, is_prefix]
+        end
+      end
+    end
+  end
+end

data/lib/iev/reconciler/termbase_loader.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Iev
+  module Reconciler
+    # Loads termbase.yaml into a raw hash, then builds
+    # Glossarist::ManagedConcept objects on demand per code.
+    # This avoids the cost of constructing 22k model objects upfront.
+    class TermbaseLoader
+      # @param path [String, Pathname] path to termbase.yaml
+      def initialize(path)
+        @path = path.to_s
+        @data = nil
+      end
+
+      # @return [Hash<String, Hash>] raw termbase data keyed by code
+      def raw
+        @data ||= YAML.load_file(@path)
+      end
+
+      # @return [Array<String>] all codes in the termbase
+      def codes
+        raw.keys.map(&:to_s)
+      end
+
+      # Build a single ManagedConcept for the given code.
+      # @param code [String]
+      # @return [Glossarist::ManagedConcept, nil]
+      def get(code)
+        entry = raw[code] || raw[code.to_sym]
+        return nil unless entry.is_a?(Hash)
+
+        build_concept(code.to_s, entry)
+      end
+
+      private
+
+      def build_concept(code, entry)
+        concept = Glossarist::ManagedConcept.of_yaml(
+          "id" => code, "data" => { "id" => code },
+        )
+        concept.status = concept_status(entry)
+        concept.dates = concept_dates(entry)
+        concept.schema_version = "3"
+
+        build_localized(code, entry).each do |lc|
+          concept.add_l10n(lc)
+        end
+
+        concept
+      end
+
+      def concept_status(entry)
+        lang_data = first_language(entry)
+        StatusMapper.call(lang_data&.dig("entry_status"))
+      end
+
+      def concept_dates(entry)
+        lang_data = first_language(entry)
+        return [] unless lang_data
+
+        dates = []
+        if lang_data["date_accepted"]
+          dates << Glossarist::ConceptDate.new(
+            type: "accepted",
+            date: lang_data["date_accepted"],
+          )
+        end
+        if lang_data["date_amended"] && lang_data["date_amended"] != lang_data["date_accepted"]
+          dates << Glossarist::ConceptDate.new(
+            type: "amended",
+            date: lang_data["date_amended"],
+          )
+        end
+        dates
+      end
+
+      def first_language(entry)
+        entry.values.find { |v| v.is_a?(Hash) && v["terms"] }
+      end
+
+      def build_localized(code, entry)
+        result = []
+        entry.each do |lang, data|
+          next unless data.is_a?(Hash) && data["terms"]
+          lc = build_localized_concept(code, lang, data)
+          result << lc if lc
+        end
+        result
+      end
+
+      def build_localized_concept(code, lang, data)
+        cdata = Glossarist::ConceptData.new
+        cdata.id = code
+        cdata.language_code = lang
+        cdata.entry_status = StatusMapper.call(data["entry_status"])
+
+        cdata.terms = (data["terms"] || []).map do |t|
+          build_term_expression(t)
+        end
+
+        if data["definition"] && !data["definition"].empty?
+          cdata.definition = [Glossarist::DetailedDefinition.new(content: data["definition"])]
+        end
+
+        cdata.notes = Array(data["notes"]).map { |n| Glossarist::DetailedDefinition.new(content: n) }
+        cdata.examples = Array(data["examples"]).map { |e| Glossarist::DetailedDefinition.new(content: e) }
+
+        lc = Glossarist::LocalizedConcept.new
+        lc.id = code
+        lc.data = cdata
+        lc
+      end
+
+      def build_term_expression(term_data)
+        designation = term_data["designation"].to_s
+        parsed = TermMarkerParser.parse(designation)
+
+        expr = Glossarist::Designation::Expression.new(
+          designation: parsed.designation,
+          normative_status: term_data["normative_status"] || "preferred",
+          type: term_data["type"] || "expression",
+        )
+        expr.geographical_area = term_data["geographical_area"] if term_data["geographical_area"]
+        expr.field_of_application = term_data["usage_info"] || parsed.domain
+        expr.usage_info = parsed.usage_info if parsed.usage_info
+        expr.prefix = parsed.is_prefix if parsed.is_prefix
+
+        grammar = Glossarist::Designation::GrammarInfo.new
+        merged_genders = (Array(term_data["gender"]) + parsed.genders).flatten.compact.uniq
+        merged_numbers = (Array(term_data["plurality"]) + parsed.numbers).flatten.compact.uniq
+        grammar.gender = merged_genders if merged_genders.any?
+        grammar.number = merged_numbers if merged_numbers.any?
+        grammar.part_of_speech = parsed.part_of_speech if parsed.part_of_speech
+        expr.grammar_info = [grammar] if grammar.gender&.any? || grammar.number&.any? || grammar.part_of_speech
+        expr
+      end
+    end
+  end
+end

data/lib/iev/reconciler.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Iev
+  module Reconciler
+    autoload :Change,            "iev/reconciler/change"
+    autoload :ChangeSet,         "iev/reconciler/change_set"
+    autoload :ConceptMerger,     "iev/reconciler/concept_merger"
+    autoload :ContentDiffer,     "iev/reconciler/content_differ"
+    autoload :LiveLoader,        "iev/reconciler/live_loader"
+    autoload :Pipeline,          "iev/reconciler/pipeline"
+    autoload :ReconciledConcept, "iev/reconciler/reconciled_concept"
+    autoload :Report,            "iev/reconciler/report"
+    autoload :StatusMapper,      "iev/reconciler/status_mapper"
+    autoload :TermMarkerParser,  "iev/reconciler/term_marker_parser"
+    autoload :TermbaseLoader,    "iev/reconciler/termbase_loader"
+  end
+end