openphar 0.1.0

data/lib/openphar/monograph_merger.rb

@@ -0,0 +1,374 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Openphar
+  # Merges Japanese content into existing English monograph JSON-LD files
+  #
+  # This class:
+  # - Loads existing English JSON-LD monograph files
+  # - Parses Japanese HTML using JpJaHtmlParser
+  # - Matches monographs by Latin name
+  # - Merges Japanese content into the existing JSON-LD structure
+  # - Preserves language tags for all properties
+  class MonographMerger
+    # Mapping of Japanese section types to JSON-LD property names
+    SECTION_PROPERTY_MAP = {
+      definition: "definition",
+      macroscopic_description: "macroscopicDescription",
+      microscopic_description: "microscopicDescription",
+      identification: "identification",
+      purity: "purity",
+      foreign_matter: "foreignMatter",
+      loss_on_drying: "lossOnDrying",
+      total_ash: "totalAsh",
+      acid_insoluble_ash: "acidInsolubleAsh",
+      extractive: "extractive",
+      assay: "assay",
+      storage: "storageConditions",
+      container: "storageContainer"
+    }.freeze
+
+    attr_reader :existing_monographs, :japanese_monographs
+
+    # Initialize the merger
+    #
+    # @param existing_dir [String] Path to directory containing existing JSON-LD files
+    # @param japanese_dir [String] Path to Japanese HTML files (optional)
+    def initialize(existing_dir: nil, japanese_dir: nil)
+      @existing_dir = existing_dir
+      @japanese_dir = japanese_dir
+      @existing_monographs = {}
+      @japanese_monographs = []
+    end
+
+    # Load existing monographs from JSON-LD files
+    #
+    # @param directory [String] Path to directory containing JSON-LD files
+    # @return [Hash] Hash of normalized Latin name => monograph data
+    def load_existing_monographs(directory = nil)
+      dir = directory || @existing_dir
+      return {} unless dir && Dir.exist?(dir)
+
+      Dir.glob(File.join(dir, "**", "*.jsonld")).each do |file|
+        load_monograph_file(file)
+      end
+
+      @existing_monographs
+    end
+
+    # Load and parse Japanese monographs
+    #
+    # @param html_file [String] Path to Japanese HTML file
+    # @return [Array<Hash>] Parsed Japanese monographs
+    def load_japanese_monographs(html_file = nil)
+      file = html_file || @japanese_dir
+      return [] unless file && File.exist?(file)
+
+      parser = Openphar::Parsers::JpJaHtmlParser.new(
+        File.read(file, encoding: "UTF-8")
+      )
+      @japanese_monographs = parser.parse
+
+      # Index by normalized Latin name (standard Ruby, no ActiveSupport)
+      @japanese_index = {}
+      @japanese_monographs.each do |m|
+        key = normalize_latin_name(m[:latin_name])
+        @japanese_index[key] = m
+      end
+      @japanese_index
+    end
+
+    # Merge Japanese content into existing monographs
+    #
+    # @param english_monograph [Hash] Existing JSON-LD monograph (as Ruby Hash)
+    # @param japanese_data [Hash] Parsed Japanese monograph data
+    # @return [Hash] Updated monograph with bilingual content
+    def merge_japanese_content(english_monograph, japanese_data)
+      return english_monograph unless japanese_data
+
+      merged = english_monograph.dup
+
+      # Add Japanese name to prefLabel
+      if japanese_data[:japanese_name]
+        merged["prefLabel"] ||= {}
+        merged["prefLabel"]["ja"] = japanese_data[:japanese_name]
+      end
+
+      # Merge Japanese content into sections
+      if japanese_data[:sections]
+        japanese_data[:sections].each do |section_type, content|
+          property_name = SECTION_PROPERTY_MAP[section_type]
+          next unless property_name
+
+          japanese_content = content.join("\n\n")
+
+          case property_name
+          when "definition"
+            merge_language_content(merged, property_name, japanese_content)
+          when "macroscopicDescription"
+            merged["macroscopicDescription"] ||= {}
+            merged["macroscopicDescription"]["ja"] = japanese_content
+          when "microscopicDescription"
+            merged["microscopicDescription"] ||= {}
+            merged["microscopicDescription"]["ja"] = japanese_content
+          when "storageConditions"
+            merge_language_content(merged, property_name, japanese_content)
+          when "identification"
+            merge_identification_tests(merged, japanese_data)
+          when "assay"
+            merge_assay_tests(merged, japanese_data)
+          when "purity"
+            merge_purity_tests(merged, japanese_data)
+          else
+            # Generic handling for other test specifications
+            merge_test_specifications(merged, section_type, japanese_content)
+          end
+        end
+      end
+
+      merged
+    end
+
+    # Merge all Japanese monographs into existing monographs
+    #
+    # @param existing_monographs [Hash] Existing monographs indexed by Latin name
+    # @param japanese_monographs [Array<Hash>] Parsed Japanese monographs
+    # @return [Array<Hash>] Updated monographs
+    def merge_all(existing_monographs = nil, japanese_monographs = nil)
+      existing = existing_monographs || @existing_monographs
+      japanese_index = @japanese_index || {}
+
+      merged_results = []
+
+      japanese_index.each do |latin_name, japanese_data|
+        english_data = existing[latin_name]
+
+        if english_data
+          merged = merge_japanese_content(english_data.dup, japanese_data)
+          merged_results << merged
+        else
+          # No matching English monograph found - could be a new monograph
+          # or one that exists only in Japanese
+          merged_results << create_japanese_only_monograph(japanese_data)
+        end
+      end
+
+      merged_results
+    end
+
+    # Write merged monographs to directory
+    #
+    # @param monographs [Array<Hash>] Merged monographs
+    # @param output_dir [String] Output directory path
+    def write_merged_monographs(monographs, output_dir)
+      FileUtils.mkdir_p(output_dir)
+
+      monographs.each do |monograph|
+        next unless monograph["@id"]
+
+        # Extract filename from IRI
+        slug = monograph["@id"].split("/").last
+        output_path = File.join(output_dir, "#{slug}.jsonld")
+
+        File.write(output_path, JSON.pretty_generate(monograph))
+      end
+    end
+
+    private
+
+    # Load a single monograph JSON-LD file
+    #
+    # @param file_path [String] Path to JSON-LD file
+    def load_monograph_file(file_path)
+      data = JSON.parse(File.read(file_path))
+
+      # Extract Latin name from prefLabel or monographId
+      latin_name = extract_latin_name_from_monograph(data)
+
+      @existing_monographs[normalize_latin_name(latin_name)] = data if latin_name
+    end
+
+    # Extract Latin name from monograph data
+    #
+    # @param data [Hash] Monograph JSON-LD data
+    # @return [String, nil] Latin name
+    def extract_latin_name_from_monograph(data)
+      # Try prefLabel first
+      label = data["prefLabel"]&.[]("en")
+      return label if label
+
+      # Fall back to monographId
+      data["monographId"]
+    end
+
+    # Normalize Latin name for matching
+    #
+    # @param name [String] Original name
+    # @return [String] Normalized name
+    def normalize_latin_name(name)
+      return nil unless name
+
+      name
+        .to_s
+        .strip
+        .upcase
+        .gsub(/\s+/, " ")
+        .gsub(/[^A-Z\s\-]/, "")
+    end
+
+    # Merge language-tagged content
+    #
+    # @param merged [Hash] The merged monograph
+    # @param property [String] Property name
+    # @param japanese_content [String] Japanese content
+    def merge_language_content(merged, property, japanese_content)
+      merged[property] ||= {}
+      merged[property]["ja"] = japanese_content
+    end
+
+    # Merge identification tests
+    #
+    # @param merged [Hash] The merged monograph
+    # @param japanese_data [Hash] Japanese data
+    def merge_identification_tests(merged, japanese_data)
+      # For identification, add Japanese content to existing test specifications
+      merged["testSpecification"] ||= []
+
+      japanese_id_tests = japanese_data[:sections][:identification]
+      return unless japanese_id_tests
+
+      japanese_content = japanese_id_tests.join("\n\n")
+
+      # Find or create identification test
+      id_test = merged["testSpecification"].find do |spec|
+        spec["testType"] == "TestType-Identification"
+      end
+
+      if id_test
+        id_test["testConditions"] ||= {}
+        id_test["testConditions"]["ja"] = japanese_content
+      end
+    end
+
+    # Merge assay tests
+    #
+    # @param merged [Hash] The merged monograph
+    # @param japanese_data [Hash] Japanese data
+    def merge_assay_tests(merged, japanese_data)
+      merged["testSpecification"] ||= []
+
+      japanese_assay = japanese_data[:sections][:assay]
+      return unless japanese_assay
+
+      japanese_content = japanese_assay.join("\n\n")
+
+      # Find or create assay test
+      assay_test = merged["testSpecification"].find do |spec|
+        spec["testType"] == "TestType-Assay"
+      end
+
+      if assay_test
+        assay_test["testConditions"] ||= {}
+        assay_test["testConditions"]["ja"] = japanese_content
+      end
+    end
+
+    # Merge purity tests
+    #
+    # @param merged [Hash] The merged monograph
+    # @param japanese_data [Hash] Japanese data
+    def merge_purity_tests(merged, japanese_data)
+      merged["testSpecification"] ||= []
+
+      japanese_purity = japanese_data[:sections][:purity]
+      return unless japanese_purity
+
+      japanese_content = japanese_purity.join("\n\n")
+
+      # Find or create purity test
+      purity_test = merged["testSpecification"].find do |spec|
+        spec["testType"] == "TestType-Purity"
+      end
+
+      if purity_test
+        purity_test["testConditions"] ||= {}
+        purity_test["testConditions"]["ja"] = japanese_content
+      end
+    end
+
+    # Generic test specification merger
+    #
+    # @param merged [Hash] The merged monograph
+    # @param section_type [Symbol] Section type
+    # @param content [String] Japanese content
+    def merge_test_specifications(merged, section_type, content)
+      merged["testSpecification"] ||= []
+
+      property_name = SECTION_PROPERTY_MAP[section_type]
+
+      # Create or update test specification
+      test_type = section_type_to_test_type(section_type)
+
+      existing_spec = merged["testSpecification"].find do |spec|
+        spec["testType"] == test_type
+      end
+
+      if existing_spec
+        existing_spec["testConditions"] ||= {}
+        existing_spec["testConditions"]["ja"] = content
+      end
+    end
+
+    # Convert section type to test type
+    #
+    # @param section_type [Symbol] Section type
+    # @return [String] Test type
+    def section_type_to_test_type(section_type)
+      case section_type
+      when :identification
+        "TestType-Identification"
+      when :purity
+        "TestType-Purity"
+      when :assay
+        "TestType-Assay"
+      when :loss_on_drying
+        "TestType-LossOnDrying"
+      when :total_ash
+        "TestType-TotalAsh"
+      when :acid_insoluble_ash
+        "TestType-AcidInsolubleAsh"
+      when :foreign_matter
+        "TestType-ForeignMatter"
+      else
+        "TestType-Physical"
+      end
+    end
+
+    # Create a Japanese-only monograph entry
+    #
+    # @param japanese_data [Hash] Japanese monograph data
+    # @return [Hash] New monograph entry
+    def create_japanese_only_monograph(japanese_data)
+      slug = japanese_data[:japanese_name]
+                    .to_s
+                    .downcase
+                    .gsub(/\s+/, "-")
+                    .gsub(/[^a-z0-9\-]/, "")
+
+      {
+        "@context" => "https://www.openphar.org/ontology/context/pharmacopoeia.jsonld",
+        "@id" => "https://www.openphar.org/data/jp/monographs/#{slug}",
+        "@type" => "CrudeDrugMonograph",
+        "prefLabel" => {
+          "ja" => japanese_data[:japanese_name],
+          "en" => japanese_data[:latin_name]
+        }.compact,
+        "publisher" => { "@id" => "op:JP" },
+        "definition" => {
+          "ja" => japanese_data[:sections][:definition]&.join("\n\n")
+        }.compact
+      }
+    end
+  end
+end

data/lib/openphar/parsers/base_monograph_parser.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Parsers
+    # Abstract base class for all monograph parsers.
+    #
+    # Defines the contract that all parsers must follow:
+    # - Return Lutaml::Model instances (not Hashes)
+    # - Track parsing statistics
+    # - Handle errors consistently
+    #
+    # @example Creating a custom parser
+    #   class MyParser < BaseMonographParser
+    #     def parse(source)
+    #       # Parse source and return model instances
+    #       MyMonograph.new(**attributes)
+    #     end
+    #   end
+    class BaseMonographParser
+      attr_reader :stats
+
+      def initialize
+        @stats = {
+          total: 0,
+          parsed: 0,
+          failed: 0,
+          errors: []
+        }
+      end
+
+      # Parse a single source (string, file content, etc.)
+      # Subclasses MUST implement this method.
+      #
+      # @param source [String] The source content to parse
+      # @return [Lutaml::Model::Serializable, nil] A model instance or nil on failure
+      # @raise [NotImplementedError] If not implemented by subclass
+      def parse(source)
+        raise NotImplementedError, 'Subclasses must implement #parse'
+      end
+
+      # Parse a single file.
+      # Subclasses MUST implement this method.
+      #
+      # @param file_path [String] Path to the file to parse
+      # @return [Lutaml::Model::Serializable, nil] A model instance or nil on failure
+      # @raise [NotImplementedError] If not implemented by subclass
+      def parse_file(file_path)
+        raise NotImplementedError, 'Subclasses must implement #parse_file'
+      end
+
+      # Parse all files in a directory.
+      # Subclasses MUST implement this method.
+      #
+      # @param directory [String] Path to the directory containing files
+      # @return [Array<Lutaml::Model::Serializable>] Array of model instances
+      # @raise [NotImplementedError] If not implemented by subclass
+      def parse_directory(directory)
+        raise NotImplementedError, 'Subclasses must implement #parse_directory'
+      end
+
+      # Reset parsing statistics
+      def reset_stats
+        @stats = {
+          total: 0,
+          parsed: 0,
+          failed: 0,
+          errors: []
+        }
+      end
+
+      protected
+
+      # Increment the total count
+      def increment_total
+        @stats[:total] += 1
+      end
+
+      # Increment the parsed count
+      def increment_parsed
+        @stats[:parsed] += 1
+      end
+
+      # Record an error with context
+      #
+      # @param source [String] The source that caused the error
+      # @param message [String] The error message
+      def record_error(source, message)
+        @stats[:failed] += 1
+        @stats[:errors] << { source: source, message: message }
+      end
+
+      # Print statistics summary
+      def print_stats_summary
+        puts "\nParsing Statistics:"
+        puts "  Total: #{@stats[:total]}"
+        puts "  Parsed: #{@stats[:parsed]}"
+        puts "  Failed: #{@stats[:failed]}"
+        return unless @stats[:errors].any?
+
+        puts "\nErrors:"
+        @stats[:errors].first(5).each do |error|
+          puts "  - #{error[:source]}: #{error[:message]}"
+        end
+        puts "  ... and #{@stats[:errors].size - 5} more" if @stats[:errors].size > 5
+      end
+    end
+  end
+end

data/lib/openphar/parsers/jp_html_parser.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Parsers
+    # Parser for Japan Pharmacopoeia HTML monographs
+    class JpHtmlParser
+      # JP monograph section patterns
+      SECTION_PATTERNS = {
+        definition: /(?:Definition|Origin and Definition)/i,
+        identification: /Identification/i,
+        purity: /Purity/i,
+        assay: /Assay/i,
+        storage: /Containers and Storage/i
+      }.freeze
+
+      attr_reader :document, :monographs
+
+      def initialize(html_content)
+        @document = Nokogiri::HTML(html_content)
+        @monographs = []
+      end
+
+      # Parse all monographs from the document
+      def parse
+        # JP HTML structure varies; this is a template
+        monograph_nodes = document.css(".monograph, .crude-drug")
+
+        monograph_nodes.each do |node|
+          monographs << parse_monograph(node)
+        end
+
+        monographs
+      end
+
+      private
+
+      def parse_monograph(node)
+        {
+          id: extract_id(node),
+          name: extract_name(node),
+          japanese_name: extract_japanese_name(node),
+          definition: extract_section(node, :definition),
+          identification: extract_section(node, :identification),
+          purity: extract_section(node, :purity),
+          assay: extract_section(node, :assay),
+          storage: extract_section(node, :storage)
+        }
+      end
+
+      def extract_id(node)
+        node.at_css(".monograph-id, .id")&.text&.strip
+      end
+
+      def extract_name(node)
+        node.at_css(".monograph-name, h1, h2")&.text&.strip
+      end
+
+      def extract_japanese_name(node)
+        node.at_css(".japanese-name, .ja")&.text&.strip
+      end
+
+      def extract_section(node, section_type)
+        pattern = SECTION_PATTERNS[section_type]
+        return nil unless pattern
+
+        section_header = node.at_css("h3, h4, .section-header") do |h|
+          h.text =~ pattern
+        end
+
+        return nil unless section_header
+
+        # Get all content until the next section header
+        content = []
+        sibling = section_header.next_element
+
+        while sibling && !section_header?(sibling)
+          content << sibling.text.strip
+          sibling = sibling.next_element
+        end
+
+        content.join("\n")
+      end
+
+      def section_header?(element)
+        return false unless element
+
+        element.name == "h3" || element.name == "h4" ||
+          element.classes.include?("section-header")
+      end
+    end
+  end
+end

data/lib/openphar/parsers/jp_html_parser_base.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Openphar
+  module Parsers
+    # Base class for JP HTML parsers
+    #
+    # Provides common functionality for parsing Japan Pharmacopoeia HTML files.
+    # Subclasses should implement:
+    # - #parse method
+    # - Section pattern constants
+    class JpHtmlParserBase
+      attr_reader :document
+
+      # Initialize parser with HTML content
+      #
+      # @param html_content [String] The HTML content to parse
+      def initialize(html_content)
+        @document = Nokogiri::HTML(html_content, nil, "UTF-8")
+      end
+
+      # Parse monographs from the document
+      # Subclasses should override this method
+      #
+      # @return [Array<Hash>] Array of parsed monograph data
+      def parse
+        raise NotImplementedError, "Subclasses must implement #parse"
+      end
+
+      # Parse monographs from a file
+      #
+      # @param file_path [String] Path to HTML file
+      # @return [Array<Hash>] Array of parsed monograph data
+      def self.parse_file(file_path)
+        html_content = File.read(file_path, encoding: "UTF-8")
+        new(html_content).parse
+      end
+
+      # Parse monographs from multiple files
+      #
+      # @param file_paths [Array<String>] Paths to HTML files
+      # @return [Array<Hash>] Combined array of parsed monograph data
+      def self.parse_files(file_paths)
+        file_paths.flat_map { |path| parse_file(path) }
+      end
+
+      protected
+
+      # Normalize Latin name for matching
+      #
+      # @param name [String] The Latin name
+      # @return [String] Normalized name
+      def normalize_latin_name(name)
+        return nil unless name
+
+        name
+          .strip
+          .upcase
+          .gsub(/\s+/, " ")
+          .gsub(/[^A-Z\s]/, "")
+      end
+
+      # Clean text content
+      #
+      # @param text [String] Raw text
+      # @return [String] Cleaned text
+      def clean_text(text)
+        return "" unless text
+
+        text
+          .strip
+          .gsub(/\s+/, " ")
+          .gsub(/[\u00A0]/, " ") # Non-breaking spaces
+      end
+
+      # Extract numeric value from text
+      #
+      # @param text [String] Text containing numeric value
+      # @return [Float, nil] Extracted numeric value
+      def extract_numeric(text)
+        match = text.match(/(\d+\.?\d*)/)
+        match ? match[1].to_f : nil
+      end
+
+      # Extract percentage from text
+      #
+      # @param text [String] Text containing percentage
+      # @return [Float, nil] Percentage value
+      def extract_percentage(text)
+        match = text.match(/(\d+\.?\d*)\s*%/)
+        match ? match[1].to_f : nil
+      end
+
+      # Extract limit from text (e.g., "not more than 1.0%", "not less than 0.5%")
+      #
+      # @param text [String] Text containing limit
+      # @return [Hash] Limit information
+      def extract_limit(text)
+        limit_type = nil
+        limit_type = :maximum if text.match?(/not more than|以下|not exceed/i)
+        limit_type = :minimum if text.match?(/not less than|以上|at least/i)
+
+        value = extract_percentage(text) || extract_numeric(text)
+
+        {
+          type: limit_type,
+          value: value
+        }.compact
+      end
+    end
+  end
+end