openphar 0.1.0

data/lib/openphar/migrators/jp_migrator.rb

@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Openphar
+  module Migrators
+    # Migrates Japan Pharmacopoeia data to Open Pharmacopoeia ontology format
+    #
+    # This class handles the transformation of JP HTML monograph files into
+    # individual JSON-LD files and aggregate files following the OP ontology.
+    class JpMigrator
+      attr_reader :source_dir, :output_dir, :edition_id
+
+      # Category mapping for organizing monographs
+      CATEGORIES = {
+        "crude_drugs" => "crude-drugs",
+        "chemical_drugs" => "chemical-drugs",
+        "formulations" => "formulations",
+        "biologicals" => "biologicals",
+        "excipients" => "excipients",
+        "vitamins" => "vitamins",
+        "amino_acids" => "amino-acids",
+        "minerals" => "minerals"
+      }.freeze
+
+      # @param source_dir [String] Directory containing JP HTML files
+      # @param output_dir [String] Output directory for JSON-LD files
+      # @param edition_id [String] Edition IRI (default: JP18)
+      def initialize(source_dir:, output_dir:, edition_id: "https://www.openphar.org/data/edition/jp/jp18")
+        @source_dir = source_dir
+        @output_dir = output_dir
+        @edition_id = edition_id
+        @monographs = []
+        @stats = { total: 0, success: 0, failed: 0, categories: Hash.new(0) }
+      end
+
+      # Run the migration
+      # @return [Hash] Statistics about the migration
+      def run
+        puts "Starting JP migration..."
+        puts "  Source: #{source_dir}"
+        puts "  Output: #{output_dir}"
+
+        # Ensure output directories exist
+        ensure_output_directories
+
+        # Process HTML files
+        process_html_files
+
+        # Generate aggregate files
+        generate_aggregate_files
+
+        # Print statistics
+        print_statistics
+
+        @stats
+      end
+
+      private
+
+      def ensure_output_directories
+        CATEGORIES.each_value do |category|
+          dir = File.join(output_dir, "monographs", category)
+          FileUtils.mkdir_p(dir)
+        end
+      end
+
+      def process_html_files
+        html_files = Dir.glob(File.join(source_dir, "**/*.html"))
+        puts "Found #{html_files.count} HTML files"
+
+        html_files.each do |html_file|
+          process_html_file(html_file)
+        end
+      end
+
+      def process_html_file(html_file)
+        puts "Processing #{html_file}..."
+
+        begin
+          html_content = File.read(html_file, encoding: "UTF-8")
+          parser = Parsers::JpHtmlParser.new(html_content)
+          parsed_monographs = parser.parse
+
+          parsed_monographs.each do |monograph_data|
+            process_monograph(monograph_data)
+          end
+        rescue StandardError => e
+          puts "  ERROR: #{e.message}"
+          puts "  #{e.backtrace.first(5).join("\n  ")}"
+          @stats[:failed] += 1
+        end
+      end
+
+      def process_monograph(data)
+        @stats[:total] += 1
+
+        # Determine category
+        category = determine_category(data)
+        @stats[:categories][category] += 1
+
+        # Generate slug from monograph name
+        slug = generate_slug(data[:name])
+
+        # Build JSON-LD structure
+        jsonld = build_monograph_jsonld(data, slug, category)
+
+        # Write individual file
+        output_path = File.join(output_dir, "monographs", category, "#{slug}.jsonld")
+        File.write(output_path, JSON.pretty_generate(jsonld))
+
+        @monographs << jsonld
+        @stats[:success] += 1
+
+        puts "  Created: #{output_path}"
+      end
+
+      def determine_category(data)
+        # Determine based on monograph type and characteristics
+        return "crude-drugs" if data[:type] == "CrudeDrugMonograph"
+        return "formulations" if data[:dosage_form]
+        return "vitamins" if data[:name]&.match?(/vitamin|ascorbic|retinol|tocopherol/i)
+        return "amino-acids" if data[:name]&.match?(/amino acid|alanine|arginine|cysteine/i)
+        return "biologicals" if data[:type] == "BiologicalSubstanceMonograph"
+        return "excipients" if data[:name]&.match?(/lactose|starch|cellulose|silicate/i)
+        return "minerals" if data[:name]&.match?(/aluminum|calcium|magnesium|zinc|iron/i)
+
+        "chemical-drugs"
+      end
+
+      def generate_slug(name)
+        return "unknown-#{@stats[:total]}" unless name
+
+        name.downcase
+            .gsub(/[^a-z0-9\s-]/, "")
+            .gsub(/\s+/, "-")
+            .gsub(/-+/, "-")
+            .strip
+            .sub(/^-/, "")
+            .sub(/-$/, "")
+      end
+
+      def build_monograph_jsonld(data, slug, category)
+        base_iri = "https://www.openphar.org/data/jp/monographs/#{category}/#{slug}"
+
+        jsonld = {
+          "@context" => "https://www.openphar.org/ontology/context/pharmacopoeia.jsonld",
+          "@id" => base_iri,
+          "@type" => data[:type] || "ChemicalDrugMonograph",
+          "prefLabel" => { "en" => data[:name] },
+          "publisher" => "JP",
+          "monographId" => data[:monograph_id],
+          "belongsToEdition" => @edition_id
+        }
+
+        # Add Japanese name if available
+        if data[:japanese_name]
+          jsonld["altLabel"] = [{ "ja" => data[:japanese_name] }]
+        end
+
+        # Add chemical properties
+        if data[:molecular_formula]
+          jsonld["molecularFormula"] = data[:molecular_formula]
+        end
+        if data[:molecular_weight]
+          jsonld["molecularWeight"] = data[:molecular_weight]
+        end
+        if data[:cas_number]
+          jsonld["casNumber"] = data[:cas_number]
+        end
+
+        # Add definition
+        if data[:definition]
+          jsonld["definition"] = { "en" => data[:definition] }
+        end
+
+        # Add appearance
+        if data[:appearance]
+          jsonld["appearance"] = { "en" => data[:appearance] }
+        end
+
+        # Add test specifications
+        if data[:tests]&.any?
+          jsonld["testSpecification"] = data[:tests].map do |test|
+            build_test_specification(test)
+          end
+        end
+
+        # Add storage
+        if data[:storage]
+          jsonld["storageConditions"] = { "en" => data[:storage] }
+        end
+
+        # Add botanical source for crude drugs
+        if data[:botanical_source]
+          jsonld["botanicalSource"] = build_botanical_source(data[:botanical_source])
+        end
+
+        # Add macroscopic description for crude drugs
+        if data[:macroscopic_description]
+          jsonld["macroscopicDescription"] = {
+            "color" => { "en" => data[:macroscopic_description][:color] },
+            "shape" => { "en" => data[:macroscopic_description][:shape] },
+            "odor" => { "en" => data[:macroscopic_description][:odor] },
+            "taste" => { "en" => data[:macroscopic_description][:taste] }
+          }.compact
+        end
+
+        jsonld.compact
+      end
+
+      def build_test_specification(test)
+        spec = {
+          "@type" => test[:spec_type] || "TestSpecification",
+          "testName" => { "en" => test[:name] }
+        }
+
+        if test[:test_type]
+          spec["testType"] = "TestType-#{test[:test_type]}"
+        end
+
+        if test[:method]
+          spec["harmonizedMethod"] = test[:method]
+        end
+
+        if test[:limit]
+          spec["limit"] = build_limit(test[:limit])
+        end
+
+        if test[:conditions]
+          spec["testConditions"] = { "en" => test[:conditions] }
+        end
+
+        spec.compact
+      end
+
+      def build_limit(limit_data)
+        limit = { "@type" => "Limit" }
+
+        if limit_data[:type]
+          limit["limitType"] = "LimitType-#{limit_data[:type]}"
+        end
+
+        if limit_data[:value]
+          limit["limitValue"] = limit_data[:value]
+        end
+
+        if limit_data[:unit]
+          limit["limitUnit"] = limit_data[:unit]
+        end
+
+        if limit_data[:lower]
+          limit["lowerLimit"] = limit_data[:lower]
+        end
+
+        if limit_data[:upper]
+          limit["upperLimit"] = limit_data[:upper]
+        end
+
+        limit.compact
+      end
+
+      def build_botanical_source(source)
+        {
+          "@type" => "BotanicalSource",
+          "plantPartUsed" => { "en" => source[:part] }
+        }.tap do |bs|
+          if source[:latin_name]
+            bs["systematicName"] = { "la" => source[:latin_name] }
+          end
+        end
+      end
+
+      def generate_aggregate_files
+        puts "\nGenerating aggregate files..."
+
+        # Generate JSON-LD aggregate
+        jsonld_aggregate = {
+          "@context" => "https://www.openphar.org/ontology/context/pharmacopoeia.jsonld",
+          "@id" => "https://www.openphar.org/data/jp/jp-monographs",
+          "@type" => "Collection",
+          "prefLabel" => { "en" => "Japanese Pharmacopoeia 18 - All Monographs" },
+          "belongsToEdition" => @edition_id,
+          "@graph" => @monographs
+        }
+
+        jsonld_path = File.join(output_dir, "jp-monographs.jsonld")
+        File.write(jsonld_path, JSON.pretty_generate(jsonld_aggregate))
+        puts "  Created: #{jsonld_path}"
+
+        # Generate TTL aggregate (via transformation)
+        ttl_content = generate_ttl_aggregate
+        ttl_path = File.join(output_dir, "jp-monographs.ttl")
+        File.write(ttl_path, ttl_content)
+        puts "  Created: #{ttl_path}"
+      end
+
+      def generate_ttl_aggregate
+        # Generate a basic Turtle representation
+        ttl = StringIO.new
+        ttl.puts "@prefix op: <https://www.openphar.org/ontology/core/> ."
+        ttl.puts "@prefix bib: <https://www.openphar.org/ontology/bibliographic/> ."
+        ttl.puts "@prefix skos: <http://www.w3.org/2004/02/skos/core#> ."
+        ttl.puts "@prefix xsd: <http://www.w3.org/2001/XMLSchema#> ."
+        ttl.puts ""
+        ttl.puts "<https://www.openphar.org/data/jp/jp-monographs>"
+        ttl.puts "    a op:Collection ;"
+        ttl.puts "    skos:prefLabel \"Japanese Pharmacopoeia 18 - All Monographs\"@en ;"
+        ttl.puts "    op:belongsToEdition <#{@edition_id}> ;"
+        ttl.puts "    op:monographCount #{@monographs.count} ."
+        ttl.puts ""
+
+        @monographs.each do |monograph|
+          id = monograph["@id"]
+          type = monograph["@type"]
+          label = monograph["prefLabel"]&.dig("en") || "Unknown"
+
+          ttl.puts "<#{id}>"
+          ttl.puts "    a op:#{type} ;"
+          ttl.puts "    skos:prefLabel #{label.inspect} ;"
+          ttl.puts "    op:publisher op:JP ;"
+          ttl.puts "    op:belongsToEdition <#{@edition_id}> ."
+          ttl.puts ""
+        end
+
+        ttl.string
+      end
+
+      def print_statistics
+        puts "\n" + "=" * 50
+        puts "Migration Statistics"
+        puts "=" * 50
+        puts "Total monographs: #{@stats[:total]}"
+        puts "Successfully migrated: #{@stats[:success]}"
+        puts "Failed: #{@stats[:failed]}"
+        puts "\nBy category:"
+        @stats[:categories].each do |category, count|
+          puts "  #{category}: #{count}"
+        end
+        puts "=" * 50
+      end
+    end
+  end
+end