openphar 0.1.0

data/lib/openphar/linkers/cross_publisher_linker.rb

@@ -0,0 +1,295 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Openphar
+  module Linkers
+    # Links pharmacopoeia monographs across different publishers
+    #
+    # This class identifies matching substances across pharmacopoeias (e.g., JP and Ph.Int.)
+    # by comparing CAS numbers, INN names, and chemical structures.
+    class CrossPublisherLinker
+      attr_reader :jp_data, :phint_data, :output_path
+
+      # Match confidence levels
+      CONFIDENCE_HIGH = 0.95   # CAS + INN match
+      CONFIDENCE_MEDIUM = 0.80 # CAS or INN match
+      CONFIDENCE_LOW = 0.60    # Name similarity only
+
+      # @param jp_data_path [String] Path to JP monographs JSON-LD file
+      # @param phint_data_path [String] Path to Ph.Int. monographs JSON-LD file
+      # @param output_path [String] Output path for cross-publisher links
+      def initialize(jp_data_path:, phint_data_path:, output_path:)
+        @jp_data_path = jp_data_path
+        @phint_data_path = phint_data_path
+        @output_path = output_path
+        @jp_monographs = []
+        @phint_monographs = []
+        @links = []
+        @stats = { total: 0, matches: 0, by_method: Hash.new(0) }
+      end
+
+      # Run the linking process
+      # @return [Hash] Statistics about the linking
+      def run
+        puts "Starting cross-publisher linking..."
+        puts "  JP data: #{@jp_data_path}"
+        puts "  Ph.Int. data: #{@phint_data_path}"
+
+        # Load data
+        load_data
+
+        # Build indices for efficient lookup
+        build_indices
+
+        # Find matches
+        find_matches
+
+        # Generate output
+        generate_links_file
+
+        # Print statistics
+        print_statistics
+
+        @stats
+      end
+
+      private
+
+      def load_data
+        puts "\nLoading data..."
+
+        # Load JP data
+        jp_content = File.read(@jp_data_path, encoding: "UTF-8")
+        jp_json = JSON.parse(jp_content)
+        @jp_monographs = extract_monographs(jp_json)
+        puts "  JP monographs: #{@jp_monographs.count}"
+
+        # Load Ph.Int. data
+        phint_content = File.read(@phint_data_path, encoding: "UTF-8")
+        phint_json = JSON.parse(phint_content)
+        @phint_monographs = extract_monographs(phint_json)
+        puts "  Ph.Int. monographs: #{@phint_monographs.count}"
+      end
+
+      def extract_monographs(data)
+        if data["@graph"]
+          data["@graph"]
+        elsif data["@id"]
+          [data]
+        else
+          []
+        end
+      end
+
+      def build_indices
+        puts "\nBuilding indices..."
+
+        # Index JP monographs by CAS number
+        @jp_by_cas = {}
+        @jp_monographs.each do |m|
+          cas = normalize_cas(m["casNumber"])
+          @jp_by_cas[cas] = m if cas
+        end
+        puts "  JP by CAS: #{@jp_by_cas.count}"
+
+        # Index JP monographs by name (INN)
+        @jp_by_name = {}
+        @jp_monographs.each do |m|
+          name = normalize_name(extract_label(m))
+          @jp_by_name[name] = m if name
+        end
+        puts "  JP by name: #{@jp_by_name.count}"
+
+        # Index Ph.Int. monographs by CAS number
+        @phint_by_cas = {}
+        @phint_monographs.each do |m|
+          cas = normalize_cas(m["casNumber"])
+          @phint_by_cas[cas] = m if cas
+        end
+        puts "  Ph.Int. by CAS: #{@phint_by_cas.count}"
+
+        # Index Ph.Int. monographs by name (INN)
+        @phint_by_name = {}
+        @phint_monographs.each do |m|
+          name = normalize_name(extract_label(m))
+          @phint_by_name[name] = m if name
+        end
+        puts "  Ph.Int. by name: #{@phint_by_name.count}"
+      end
+
+      def find_matches
+        puts "\nFinding matches..."
+
+        @jp_monographs.each do |jp_m|
+          @stats[:total] += 1
+          jp_id = jp_m["@id"]
+          jp_cas = normalize_cas(jp_m["casNumber"])
+          jp_name = normalize_name(extract_label(jp_m))
+
+          # Try CAS match first (highest confidence)
+          if jp_cas && @phint_by_cas[jp_cas]
+            phint_m = @phint_by_cas[jp_cas]
+            add_link(jp_m, phint_m, "sameSubstanceAs", CONFIDENCE_HIGH, "CAS")
+            next
+          end
+
+          # Try name match
+          if jp_name && @phint_by_name[jp_name]
+            phint_m = @phint_by_name[jp_name]
+            add_link(jp_m, phint_m, "sameSubstanceAs", CONFIDENCE_MEDIUM, "INN")
+            next
+          end
+
+          # Try fuzzy name matching for remaining
+          phint_m = find_fuzzy_match(jp_name)
+          if phint_m
+            add_link(jp_m, phint_m, "similarTo", CONFIDENCE_LOW, "FuzzyName")
+          end
+        end
+      end
+
+      def add_link(jp_m, phint_m, relation, confidence, method)
+        @links << {
+          "jp_id" => jp_m["@id"],
+          "phint_id" => phint_m["@id"],
+          "jp_name" => extract_label(jp_m),
+          "phint_name" => extract_label(phint_m),
+          "relation" => relation,
+          "confidence" => confidence,
+          "method" => method
+        }
+        @stats[:matches] += 1
+        @stats[:by_method][method] += 1
+      end
+
+      def find_fuzzy_match(jp_name)
+        return nil unless jp_name
+
+        # Simple fuzzy matching - find names with high similarity
+        @phint_by_name.each do |phint_name, phint_m|
+          next unless phint_name
+
+          # Calculate simple similarity
+          similarity = calculate_similarity(jp_name, phint_name)
+          return phint_m if similarity > 0.85
+        end
+
+        nil
+      end
+
+      def calculate_similarity(str1, str2)
+        return 1.0 if str1 == str2
+        return 0.0 if str1.nil? || str2.nil?
+
+        # Levenshtein distance-based similarity
+        distance = levenshtein_distance(str1, str2)
+        max_len = [str1.length, str2.length].max
+        1.0 - (distance.to_f / max_len)
+      end
+
+      def levenshtein_distance(str1, str2)
+        matrix = Array.new(str1.length + 1) { Array.new(str2.length + 1, 0) }
+
+        (0..str1.length).each { |i| matrix[i][0] = i }
+        (0..str2.length).each { |j| matrix[0][j] = j }
+
+        (1..str1.length).each do |i|
+          (1..str2.length).each do |j|
+            cost = str1[i - 1] == str2[j - 1] ? 0 : 1
+            matrix[i][j] = [
+              matrix[i - 1][j] + 1,      # deletion
+              matrix[i][j - 1] + 1,      # insertion
+              matrix[i - 1][j - 1] + cost # substitution
+            ].min
+          end
+        end
+
+        matrix[str1.length][str2.length]
+      end
+
+      def normalize_cas(cas)
+        return nil unless cas
+
+        # Normalize CAS format: XXXXX-XX-X
+        cas = cas.to_s.strip.upcase
+        # Remove any non-standard characters
+        cas = cas.gsub(/[^0-9\-]/, "")
+        cas.empty? ? nil : cas
+      end
+
+      def normalize_name(name)
+        return nil unless name
+
+        # Normalize drug name for comparison
+        name = name.to_s.strip.downcase
+        # Remove common suffixes that differ between pharmacopoeias
+        name = name.gsub(/\s+(hydrate|hydrochloride|sulfate|sodium|potassium)$/, "")
+        # Remove extra spaces
+        name = name.gsub(/\s+/, " ")
+        name.empty? ? nil : name
+      end
+
+      def extract_label(monograph)
+        label = monograph["prefLabel"]
+        return nil unless label
+
+        label.is_a?(Hash) ? (label["en"] || label.values.first) : label
+      end
+
+      def generate_links_file
+        puts "\nGenerating links file..."
+
+        # Ensure output directory exists
+        FileUtils.mkdir_p(File.dirname(@output_path))
+
+        # Build JSON-LD structure
+        jsonld = {
+          "@context" => "https://www.openphar.org/ontology/context/pharmacopoeia.jsonld",
+          "@id" => "https://www.openphar.org/data/cross-publisher/links",
+          "@type" => "Collection",
+          "prefLabel" => { "en" => "Cross-Publisher Links - JP and Ph.Int." },
+          "linkCount" => @links.count,
+          "@graph" => @links.map do |link|
+            {
+              "@id" => "https://www.openphar.org/data/cross-publisher/link/#{generate_link_id(link)}",
+              "@type" => "CrossPublisherLink",
+              "source" => link["jp_id"],
+              "target" => link["phint_id"],
+              "relation" => link["relation"],
+              "matchConfidence" => link["confidence"],
+              "matchMethod" => link["method"],
+              "sourceName" => link["jp_name"],
+              "targetName" => link["phint_name"]
+            }
+          end
+        }
+
+        File.write(@output_path, JSON.pretty_generate(jsonld))
+        puts "  Created: #{@output_path}"
+      end
+
+      def generate_link_id(link)
+        # Generate a unique ID for the link
+        jp_slug = link["jp_id"].split("/").last
+        phint_slug = link["phint_id"].split("/").last
+        "#{jp_slug}-#{phint_slug}"
+      end
+
+      def print_statistics
+        puts "\n" + "=" * 50
+        puts "Cross-Publisher Linking Statistics"
+        puts "=" * 50
+        puts "Total JP monographs processed: #{@stats[:total]}"
+        puts "Matches found: #{@stats[:matches]}"
+        puts "Match rate: #{(@stats[:matches].to_f / @stats[:total] * 100).round(1)}%"
+        puts "\nBy method:"
+        @stats[:by_method].each do |method, count|
+          puts "  #{method}: #{count}"
+        end
+        puts "=" * 50
+      end
+    end
+  end
+end

data/lib/openphar/linkers/herbapedia_linker.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Linkers
+    # Links pharmacopoeia monographs to data-herbapedia entities
+    class HerbapediaLinker
+      attr_reader :preparations_index, :tcm_profiles_index, :ayurveda_profiles_index
+
+      def initialize(herbapedia_data_path: nil)
+        @herbapedia_data_path = herbapedia_data_path
+        @preparations_index = {}
+        @tcm_profiles_index = {}
+        @ayurveda_profiles_index = {}
+        load_indexes if herbapedia_data_path
+      end
+
+      # Find a HerbalPreparation in data-herbapedia
+      # @param latin_name [String] The Latin pharmaceutical name
+      # @param chinese_name [String, nil] Optional Chinese name
+      # @return [String, nil] The IRI of the matching preparation
+      def find_preparation(latin_name:, chinese_name: nil)
+        return nil if preparations_index.empty?
+
+        # Try exact Latin name match first
+        normalized = normalize_name(latin_name)
+        match = preparations_index[normalized]
+
+        # Try Chinese name if no match
+        if !match && chinese_name
+          match = preparations_index.values.find do |prep|
+            prep[:names]&.any? { |n| n[:zh] == chinese_name }
+          end
+        end
+
+        match&.dig(:iri)
+      end
+
+      # Find a TCM profile in data-herbapedia
+      # @param pinyin [String] The pinyin name
+      # @return [String, nil] The IRI of the matching TCM profile
+      def find_tcm_profile(pinyin:)
+        return nil if tcm_profiles_index.empty?
+
+        normalized = normalize_name(pinyin)
+        tcm_profiles_index[normalized]&.dig(:iri)
+      end
+
+      # Find an Ayurveda profile in data-herbapedia
+      # @param sanskrit [String] The Sanskrit name
+      # @return [String, nil] The IRI of the matching Ayurveda profile
+      def find_ayurveda_profile(sanskrit:)
+        return nil if ayurveda_profiles_index.empty?
+
+        normalized = normalize_name(sanskrit)
+        ayurveda_profiles_index[normalized]&.dig(:iri)
+      end
+
+      # Link a monograph to herbapedia entities
+      # @param monograph_data [Hash] The monograph data
+      # @return [Hash] Updated monograph data with herbapedia links
+      def link_monograph(monograph_data)
+        result = monograph_data.dup
+
+        # Try to find preparation by Latin name
+        latin_name = monograph_data[:name] || monograph_data.dig(:pref_label, "en")
+        chinese_name = monograph_data.dig(:pref_label, "zh") ||
+                       monograph_data.dig(:pref_label, "zh-Hant")
+
+        if latin_name
+          prep_iri = find_preparation(latin_name: latin_name, chinese_name: chinese_name)
+          result[:references_preparation] = prep_iri if prep_iri
+        end
+
+        result
+      end
+
+      private
+
+      def load_indexes
+        # Load preparations index
+        prep_file = File.join(@herbapedia_data_path, "preparations_index.json")
+        if File.exist?(prep_file)
+          data = JSON.parse(File.read(prep_file))
+          data.each do |prep|
+            normalized = normalize_name(prep["latin_name"])
+            @preparations_index[normalized] = {
+              iri: prep["iri"],
+              names: prep["names"]
+            }
+          end
+        end
+
+        # Load TCM profiles index
+        tcm_file = File.join(@herbapedia_data_path, "tcm_profiles_index.json")
+        if File.exist?(tcm_file)
+          data = JSON.parse(File.read(tcm_file))
+          data.each do |profile|
+            normalized = normalize_name(profile["pinyin"])
+            @tcm_profiles_index[normalized] = {
+              iri: profile["iri"]
+            }
+          end
+        end
+
+        # Load Ayurveda profiles index
+        ayu_file = File.join(@herbapedia_data_path, "ayurveda_profiles_index.json")
+        if File.exist?(ayu_file)
+          data = JSON.parse(File.read(ayu_file))
+          data.each do |profile|
+            normalized = normalize_name(profile["sanskrit"])
+            @ayurveda_profiles_index[normalized] = {
+              iri: profile["iri"]
+            }
+          end
+        end
+      end
+
+      def normalize_name(name)
+        return nil unless name
+
+        name.to_s.downcase.strip.gsub(/\s+/, " ")
+      end
+    end
+  end
+end

data/lib/openphar/linkers.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Linkers
+    # Linkers for connecting monographs to external resources.
+    #
+    # Handles linking to Herbapedia entities and cross-publisher matching.
+
+    autoload :HerbapediaLinker, 'openphar/linkers/herbapedia_linker'
+    autoload :CrossPublisherLinker, 'openphar/linkers/cross_publisher_linker'
+    autoload :Chp, 'openphar/linkers/chp'
+  end
+end

data/lib/openphar/migrators/chp/classifier.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Migrators
+    module Chp
+      # Single dispatch point mapping a ChP API record's (book_id,
+      # directory_title) pair to the correct Openphar::Models::Chp::*
+      # subclass.
+      #
+      # Routing is data-driven: SPECIFIC_RULES is an ordered list of
+      # `{book_id, substring, klass_name}` triples. Adding a new directory
+      # type = adding one line, not editing dispatch logic. Open/closed.
+      #
+      # MECE invariant: every (book_id, directory_title) pair observed in
+      # the ChP 2025/2020 manifests classifies to exactly one subclass.
+      # `spec/openphar/migrators/chp/classifier_spec.rb` enforces this
+      # against every directory_title seen in the manifests.
+      #
+      class Classifier
+        SPECIFIC_RULES = [
+          { book_id: 1, substring: '药材和饮片',           klass: 'TcmCrudeDrug' },
+          { book_id: 1, substring: '植物油脂和提取物',     klass: 'TcmExtract' },
+          { book_id: 1, substring: '成方制剂',             klass: 'TcmFormulation' },
+          { book_id: 2, substring: '第二部分',             klass: 'ChemicalPreparation' },
+        ].freeze
+
+        VOLUME_DEFAULTS = {
+          1 => 'TcmCrudeDrug',
+          2 => 'ChemicalSubstance',
+          3 => 'Biologic',
+          4 => 'GeneralChapter',
+        }.freeze
+
+        FINAL_FALLBACK = 'Monograph'.freeze
+
+        class << self
+          # Returns the subclass appropriate for the given record.
+          # Never raises — degrades gracefully to Chp::Monograph.
+          #
+          # @param book_id [Integer] 1..4
+          # @param directory_title [String, nil] e.g., "药材和饮片"
+          # @return [Class] a subclass of Openphar::Models::Chp::Monograph
+          def classify(book_id:, directory_title: nil)
+            klass_name = resolve(book_id: book_id, directory_title: directory_title)
+            Openphar::Models::Chp.const_get(klass_name)
+          end
+
+          # Returns the wire_key string for a record. Useful for routing
+          # without loading the class (e.g., for stats before instantiation).
+          #
+          # @return [String]
+          def wire_key_for(book_id:, directory_title: nil)
+            classify(book_id: book_id, directory_title: directory_title).wire_key
+          end
+
+          # Enumerate every ChP type class. Used by specs to assert
+          # classification coverage.
+          #
+          # @return [Array<Class>]
+          def all_types
+            %i[
+              TcmCrudeDrug TcmExtract TcmFormulation
+              ChemicalSubstance ChemicalPreparation
+              Biologic GeneralChapter
+            ].map { |k| Openphar::Models::Chp.const_get(k) }
+          end
+
+          private
+
+          def resolve(book_id:, directory_title:)
+            return FINAL_FALLBACK unless VOLUME_DEFAULTS.key?(book_id)
+
+            SPECIFIC_RULES.each do |rule|
+              next unless rule[:book_id] == book_id
+              next if directory_title.nil? || directory_title.empty?
+
+              return rule[:klass] if directory_title.include?(rule[:substring])
+            end
+
+            VOLUME_DEFAULTS[book_id]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/openphar/migrators/chp.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Migrators
+    # Chinese Pharmacopoeia migrator namespace.
+    #
+    # Houses the classifier (single dispatch point) and the migrator that
+    # walks `data/<edition>/entries/*.json` and produces typed
+    # `Openphar::Models::Chp::*` instances + JSON-LD output.
+    #
+    module Chp
+      autoload :Classifier, 'openphar/migrators/chp/classifier'
+    end
+  end
+end

data/lib/openphar/migrators/chp_migrator.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Openphar
+  module Migrators
+    # Orchestrates the ChP migration end-to-end from the openphar gem side.
+    #
+    # Reads raw API JSON from a ChP edition's data directory, classifies +
+    # instantiates each entry to the correct Openphar::Models::Chp::* subclass,
+    # and writes JSON-LD output. Used by any consumer of the openphar gem that
+    # wants to publish ChP data without depending on the data-china-pharmacopoeia
+    # repo's CLI directly.
+    #
+    # The actual parsing (Nokogiri htmlContent → sections) is delegated to
+    # ChP::Parser when running inside the data repo. When used from another
+    # consumer, the caller supplies a parser lambda via the `parser:` keyword.
+    #
+    class ChpMigrator
+      attr_reader :edition_dir, :output_dir, :edition_label, :stats
+
+      # @param edition_dir [String] Path to the edition's data directory
+      #   (must contain `entries/*.json`).
+      # @param output_dir [String] Where to write JSON-LD output.
+      # @param edition_label [String] Harmonized label, e.g., "ChP-2025".
+      # @param parser [#call] Lambda/proc that takes (record_json, edition_label)
+      #   and returns an Openphar::Models::Chp::* instance. Defaults to
+      #   ChP::Parser method object when running inside the data repo.
+      def initialize(edition_dir:, output_dir:, edition_label:, parser: default_parser)
+        @edition_dir = edition_dir
+        @output_dir = output_dir
+        @edition_label = edition_label
+        @parser = parser
+        @stats = {
+          total: 0, classified: 0, skipped: 0, exported: 0, by_type: Hash.new(0),
+        }
+      end
+
+      def run
+        ensure_output_dirs
+        walk_entries { |monograph| export_monograph(monograph) }
+        write_aggregates
+        @stats
+      end
+
+      private
+
+      def ensure_output_dirs
+        FileUtils.mkdir_p(File.join(@output_dir, "nodes"))
+      end
+
+      def walk_entries
+        Dir.glob(File.join(@edition_dir, "entries", "*.json")).sort.each do |path|
+          @stats[:total] += 1
+          record = JSON.parse(File.read(path))
+          data = record["data"]
+
+          if data.nil?
+            @stats[:skipped] += 1
+            next
+          end
+
+          begin
+            monograph = @parser.call(record, @edition_label)
+            yield monograph
+            @stats[:classified] += 1
+            @stats[:by_type][monograph.class.wire_key] += 1
+          rescue StandardError => e
+            warn "[#{@edition_label}] #{path}: #{e.message}"
+            @stats[:skipped] += 1
+          end
+        end
+      end
+
+      def export_monograph(monograph)
+        dir = File.join(@output_dir, "nodes", monograph.class.wire_key)
+        FileUtils.mkdir_p(dir)
+        path = File.join(dir, monograph.jsonld_filename)
+        doc = { "@context" => context_url }.merge(monograph_to_jsonld(monograph))
+        File.write(path, JSON.pretty_generate(doc))
+        @stats[:exported] += 1
+      end
+
+      def write_aggregates
+        # Placeholder for index/full graph generation — the data repo's exporter
+        # is more complete; this gem-side migrator focuses on per-node output.
+        File.write(File.join(@output_dir, "migrator-stats.json"),
+                   JSON.pretty_generate(@stats))
+      end
+
+      def monograph_to_jsonld(m)
+        {
+          "@id" => m.id || m.iri,
+          "@type" => m.class.jsonld_types,
+          "monographId" => m.monograph_id,
+          "belongsToEdition" => m.belongs_to_edition,
+          "prefLabel" => m.pref_label,
+          "publisher" => m.publisher,
+          "status" => m.status,
+          "definition" => m.definition,
+          "entryId" => m.entry_id,
+          "bookId" => m.book_id,
+          "directoryTitle" => m.directory_title,
+          "title" => m.title,
+          "pinyinTitle" => m.pinyin_title,
+          "eTitle" => m.e_title,
+          "pageNum" => m.page_num,
+        }.compact
+      end
+
+      def context_url
+        "https://www.openphar.org/ontology/context/chp.jsonld"
+      end
+
+      def default_parser
+        lambda do |record, edition_label|
+          edition_key = edition_label.to_s.downcase.gsub(/\Achp-/, "")
+          ChP::Parser.parse_record(record, edition: edition_key)
+        end
+      rescue NameError
+        # ChP::Parser not loadable (running outside the data repo) — the caller
+        # must supply an explicit parser lambda.
+        lambda { |*, |
+          raise "no default parser available; supply `parser:` keyword to ChpMigrator.new"
+        }
+      end
+    end
+  end
+end