openphar 0.1.0

data/lib/openphar/exporters/coverage_validator.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require "json"
+require "set"
+
+module Openphar
+  module Exporters
+    # Validates that model coverage is 100% complete
+    #
+    # Compares:
+    # 1. All fields in JSON-LD files have corresponding model attributes
+    # 2. All model attributes can be exported to JSON-LD
+    # 3. All model attributes can be exported to Neo4j
+    class CoverageValidator
+      # Standard JSON-LD keys that are framework-related, not domain data
+      STANDARD_KEYS = %w[
+        @context @graph @id @type @vocab @base
+      ].freeze
+
+      # Mapping from JSON-LD camelCase to model snake_case
+      FIELD_MAPPING = {
+        "monographId" => "monograph_id",
+        "prefLabel" => "pref_label",
+        "altLabel" => "alt_label",
+        "belongsToEdition" => "belongs_to_edition",
+        "effectiveDate" => "effective_date",
+        "referencesPreparation" => "references_preparation",
+        "referencesTCMProfile" => "references_tcm_profile",
+        "referencesAyurvedaProfile" => "references_ayurveda_profile",
+        "referencesWesternProfile" => "references_western_profile",
+        "referencesPlantSpecies" => "references_plant_species",
+        "referencesPlantPart" => "references_plant_part",
+        "testSpecification" => "test_specifications",
+        "sameSubstanceAs" => "same_substance_as",
+        "hasEquivalentIn" => "has_equivalent_in",
+        "similarTo" => "similar_to",
+        "storageContainer" => "storage_container",
+        "storageConditions" => "storage_conditions",
+        "botanicalSource" => "botanical_source",
+        "macroscopicDescription" => "macroscopic_description",
+        "microscopicDescription" => "microscopic_description",
+        "molecularFormula" => "molecular_formula",
+        "molecularWeight" => "molecular_weight",
+        "casNumber" => "cas_number",
+        "systematicName" => "systematic_name",
+        "inchiKey" => "inchi_key",
+        "smiles" => "smiles",
+        "chemicalStructure" => "chemical_structure",
+        "appearance" => "appearance",
+        "solubility" => "solubility",
+        "testName" => "test_name",
+        "testType" => "test_type",
+        "harmonizedMethod" => "harmonized_method",
+        "publisherMethod" => "publisher_method",
+        "testConditions" => "test_conditions",
+        "limitType" => "limit_type",
+        "limitValue" => "limit_value",
+        "lowerLimit" => "lower_limit",
+        "upperLimit" => "upper_limit",
+        "limitUnit" => "limit_unit",
+        "assayTarget" => "assay_target",
+        "assayExpression" => "assay_expression",
+        "impurityType" => "impurity_type",
+        "physicalProperty" => "physical_property"
+      }.freeze
+
+      # Reverse mapping: snake_case to camelCase
+      REVERSE_MAPPING = FIELD_MAPPING.invert
+
+      attr_reader :results, :missing_fields, :extra_fields
+
+      def initialize
+        @results = {}
+        @missing_fields = []
+        @extra_fields = []
+      end
+
+      # Analyze a JSON-LD file and compare to expected model attributes
+      #
+      # @param jsonld_file [String] Path to JSON-LD file
+      # @param expected_fields [Array<Symbol>] Expected field names from model
+      def validate_file(jsonld_file, expected_fields = nil)
+        data = JSON.parse(File.read(jsonld_file))
+
+        # Extract all keys from JSON-LD
+        jsonld_keys = extract_all_keys(data).to_a
+        domain_keys = jsonld_keys - STANDARD_KEYS
+
+        # Normalize keys to snake_case for comparison
+        normalized_jsonld_keys = domain_keys.map { |k| normalize_key(k) }
+
+        # Get expected fields from model if not provided
+        expected ||= expected_fields || model_expected_fields
+        expected_normalized = expected.map(&:to_s)
+
+        # Find missing fields (in JSON-LD but not in model)
+        missing = normalized_jsonld_keys - expected_normalized
+
+        # Find extra fields (in model but not in JSON-LD)
+        extra = expected_normalized - normalized_jsonld_keys
+
+        @missing_fields.concat(missing)
+        @extra_fields.concat(extra)
+
+        coverage = if normalized_jsonld_keys.any?
+                     (1 - missing.size.to_f / normalized_jsonld_keys.size) * 100
+                   else
+                     100
+                   end
+
+        @results[jsonld_file] = {
+          jsonld_keys: domain_keys.size,
+          expected: expected.size,
+          missing: missing,
+          extra: extra,
+          coverage: coverage
+        }
+
+        { missing: missing, extra: extra, coverage: coverage }
+      end
+
+      # Normalize a key from JSON-LD to snake_case
+      def normalize_key(key)
+        # Remove nested path (e.g., "prefLabel.en" -> "prefLabel")
+        base_key = key.split(".").first
+
+        # Map to snake_case if known
+        FIELD_MAPPING[base_key] || base_key
+      end
+
+      # Validate all JSON-LD files in a directory
+      #
+      # @param directory [String] Directory path
+      # @param expected_fields [Array<Symbol>] Expected model fields
+      def validate_directory(directory, expected_fields = nil)
+        Dir.glob(File.join(directory, "**", "*.jsonld")).each do |file|
+          puts "Validating: #{file}"
+          validate_file(file, expected_fields)
+        end
+
+        summary
+      end
+
+      # Generate report of all unique missing fields across all files
+      #
+      # @return [Hash] Report of coverage
+      def summary
+        unique_missing = @missing_fields.uniq
+        unique_extra = @extra_fields.uniq
+
+        total_jsonld_keys = @results.values.sum { |r| r[:jsonld_keys] }
+        total_expected = @results.values.sum { |r| r[:expected] }
+
+        avg_coverage = if @results.any?
+                         @results.values.map { |r| r[:coverage] }.sum / @results.size
+                       else
+                         100
+                       end
+
+        {
+          files_validated: @results.size,
+          total_jsonld_fields: total_jsonld_keys,
+          total_expected_fields: total_expected,
+          unique_missing_fields: unique_missing,
+          unique_extra_fields: unique_extra,
+          average_coverage: avg_coverage,
+          details: @results
+        }
+      end
+
+      # Print detailed report
+      def print_report
+        puts "=" * 60
+        puts "COVERAGE VALIDATION REPORT"
+        puts "=" * 60
+
+        summary = self.summary
+
+        puts "\nFiles Validated: #{summary[:files_validated]}"
+        puts "Total JSON-LD Fields: #{summary[:total_jsonld_fields]}"
+        puts "Total Expected Fields: #{summary[:total_expected_fields]}"
+        puts "Average Coverage: #{summary[:average_coverage].round(2)}%"
+
+        if summary[:unique_missing_fields].any?
+          puts "\n" + "!" * 60
+          puts "MISSING FIELDS (in JSON-LD but not in model):"
+          puts "!" * 60
+          summary[:unique_missing_fields].each do |field|
+            puts "  - #{field}"
+          end
+        end
+
+        if summary[:unique_extra_fields].any?
+          puts "\n" + "+" * 60
+          puts "EXTRA FIELDS (in model but not in JSON-LD):"
+          puts "+" * 60
+          summary[:unique_extra_fields].each do |field|
+            puts "  + #{field}"
+          end
+        end
+
+        puts "\n" + "=" * 60
+        puts "COVERAGE BY FILE:"
+        puts "=" * 60
+
+        @results.each do |file, result|
+          status = result[:coverage] >= 100 ? "✓" : "✗"
+          puts "#{status} #{File.basename(file)}: #{result[:coverage].round(2)}%"
+        end
+      end
+
+      private
+
+      # Extract ALL keys from JSON-LD data (recursive)
+      def extract_all_keys(data, prefix = nil)
+        keys = Set.new
+
+        case data
+        when Hash
+          data.each do |key, value|
+            full_key = prefix ? "#{prefix}.#{key}" : key.to_s
+            keys << full_key
+
+            # Recurse into nested structures
+            case value
+            when Hash
+              keys.merge(extract_all_keys(value, full_key))
+            when Array
+              value.each do |item|
+                keys.merge(extract_all_keys(item, full_key)) if item.is_a?(Hash)
+              end
+            end
+          end
+        when Array
+          data.each do |item|
+            keys.merge(extract_all_keys(item, prefix)) if item.is_a?(Hash)
+          end
+        end
+
+        keys
+      end
+
+      # Expected fields from our models
+      def model_expected_fields
+        # This matches the fields in our Lutaml::Model classes
+        %i[
+          id type monograph_id pref_label alt_label definition publisher
+          status belongs_to_edition effective_date version references_preparation
+          references_tcm_profile references_ayurveda_profile references_western_profile
+          references_plant_species test_specifications same_substance_as
+          has_equivalent_in similar_to storage_container storage_conditions
+          botanical_source macroscopic_description microscopic_description
+          molecular_formula molecular_weight cas_number systematic_name
+          inchi inchi_key smiles appearance solubility chemical_structure
+          test_name test_type harmonized_method publisher_method test_conditions
+          limit_type limit_value lower_limit upper_limit limit_unit
+          assay_target assay_expression impurity_type physical_property
+          color shape texture odor taste fracture size
+          transverse_section powder_characteristics diagnostic_features
+          references_plant_part collection_period
+        ]
+      end
+    end
+  end
+end

data/lib/openphar/exporters/neo4j/model_registry.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Exporters
+    module Neo4j
+      # Registry of all models that can be exported to Neo4j
+      #
+      # Provides mapping from JSON-LD @type to:
+      # - Neo4j label(s)
+      # - Ruby model class
+      # - Serialization key mappings
+      #
+      # This registry is the central source of truth for model-to-Neo4j mapping.
+      class ModelRegistry
+        # Node type mappings: JSON-LD @type => Neo4j labels
+        NODE_TYPES = {
+          'PharmacopoeiaMonograph' => 'Monograph',
+          'CrudeDrugMonograph' => 'Monograph:CrudeDrug',
+          'ChemicalDrugMonograph' => 'Monograph:ChemicalDrug',
+          'FormulationMonograph' => 'Monograph:Formulation',
+          'VitaminMonograph' => 'Monograph:Vitamin',
+          'AminoAcidMonograph' => 'Monograph:AminoAcid',
+          'MineralSubstanceMonograph' => 'Monograph:Mineral',
+          'BiologicalSubstanceMonograph' => 'Monograph:Biological',
+          # JP-specific types
+          'KampoFormula' => 'Monograph:Formulation:Kampo',
+          # Ph.Int.-specific types
+          'RadiopharmaceuticalMonograph' => 'Monograph:Radiopharmaceutical',
+          'DosageFormMonograph' => 'Monograph:DosageForm',
+          'TestMethod' => 'TestMethod',
+          'Reagent' => 'Reagent',
+          'TestSolution' => 'Reagent:TestSolution',
+          'VolumetricSolution' => 'Reagent:VolumetricSolution',
+          'BufferSolution' => 'Reagent:BufferSolution',
+          'ReferenceSubstance' => 'ReferenceSubstance',
+          # Test specifications
+          'TestSpecification' => 'TestSpec',
+          'AssaySpecification' => 'TestSpec:Assay',
+          'PuritySpecification' => 'TestSpec:Purity',
+          'IdentificationSpecification' => 'TestSpec:Identity',
+          'PhysicalSpecification' => 'TestSpec:Physical',
+          # Other
+          'Limit' => 'Limit',
+          'Edition' => 'Edition',
+          'Supplement' => 'Supplement',
+          'Publisher' => 'Publisher'
+        }.freeze
+
+        # Relationship types
+        RELATIONSHIP_TYPES = {
+          'sameSubstanceAs' => 'SAME_SUBSTANCE_AS',
+          'hasEquivalentIn' => 'HAS_EQUIVALENT_IN',
+          'similarTo' => 'SIMILAR_TO',
+          'belongsToEdition' => 'BELONGS_TO_EDITION',
+          'hasTestSpec' => 'HAS_TEST_SPEC',
+          'referencesPreparation' => 'REFERENCES_PREPARATION',
+          'referencesTCMProfile' => 'REFERENCES_TCM_PROFILE'
+        }.freeze
+
+        # Returns Neo4j labels for a given JSON-LD @type
+        #
+        # @param jsonld_type [String] The @type from JSON-LD
+        # @return [String] Neo4j label(s)
+        def node_type_for(jsonld_type)
+          NODE_TYPES[jsonld_type] || 'Monograph'
+        end
+
+        # Returns the Ruby model class for a given JSON-LD @type
+        #
+        # @param jsonld_type [String] The @type from JSON-LD
+        # @return [Class, nil] The model class
+        def model_class_for(jsonld_type)
+          entry = MODEL_CLASSES[jsonld_type]
+          entry&.call
+        end
+
+        # Returns the Neo4j relationship type for a given link type
+        #
+        # @param link_type [String] The relationship type from JSON-LD
+        # @return [String] Neo4j relationship type
+        def relationship_type_for(link_type)
+          RELATIONSHIP_TYPES[link_type] || 'RELATED_TO'
+        end
+
+        # Returns all known node types
+        #
+        # @return [Array<String>] List of Neo4j labels
+        def all_node_types
+          NODE_TYPES.values.flat_map { |t| t.split(':') }.uniq
+        end
+
+        # Checks if a given type is a monograph type
+        #
+        # @param jsonld_type [String] The @type from JSON-LD
+        # @return [Boolean] True if it's a monograph type
+        def monograph_type?(jsonld_type)
+          NODE_TYPES[jsonld_type]&.include?('Monograph')
+        end
+
+        # Model classes indexed by JSON-LD @type
+        # Uses lazy loading to avoid circular dependency issues
+        MODEL_CLASSES = {
+          # Core monograph types
+          'CrudeDrugMonograph' => lambda { Openphar::Models::CrudeDrugMonograph },
+          'ChemicalDrugMonograph' => lambda { Openphar::Models::ChemicalDrugMonograph },
+          'Monograph' => lambda { Openphar::Models::Monograph },
+          # Bibliographic
+          'Edition' => lambda { Openphar::Models::Edition },
+          'Supplement' => lambda { Openphar::Models::Supplement },
+          # Test specifications
+          'TestSpecification' => lambda { Openphar::Models::TestSpecification },
+          'AssaySpecification' => lambda { Openphar::Models::AssaySpecification },
+          'PuritySpecification' => lambda { Openphar::Models::PuritySpecification },
+          'PhysicalSpecification' => lambda { Openphar::Models::PhysicalSpecification },
+          # JP models (in JP namespace)
+          'KampoFormula' => lambda { Openphar::Models::JP::KampoFormula },
+          # Ph.Int. models (in PhInt namespace)
+          'RadiopharmaceuticalMonograph' => lambda { Openphar::Models::PhInt::RadiopharmaceuticalMonograph },
+          'DosageFormMonograph' => lambda { Openphar::Models::PhInt::DosageFormMonograph },
+          'TestMethod' => lambda { Openphar::Models::PhInt::TestMethod },
+          'Reagent' => lambda { Openphar::Models::PhInt::Reagent },
+          'TestSolution' => lambda { Openphar::Models::PhInt::TestSolution },
+          'VolumetricSolution' => lambda { Openphar::Models::PhInt::VolumetricSolution },
+          'BufferSolution' => lambda { Openphar::Models::PhInt::BufferSolution },
+          'ReferenceSubstance' => lambda { Openphar::Models::PhInt::ReferenceSubstance }
+        }.freeze
+      end
+    end
+  end
+end

data/lib/openphar/exporters/neo4j/node_builder.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Exporters
+    module Neo4j
+      # Builds Neo4j Cypher nodes from model data
+      #
+      # Uses the property mapper to convert JSON-LD properties to Neo4j format,
+      # and generates either CREATE or MERGE statements based on the use_merge flag.
+      class NodeBuilder
+        attr_reader :registry, :mapper
+
+        def initialize(mapper = nil)
+          @mapper = mapper || PropertyMapper.new
+          @registry = @mapper.instance_variable_get(:@registry) || ModelRegistry.new
+        end
+
+        # Builds a Cypher CREATE/MERGE statement for a single node
+        #
+        # @param data [Hash] The JSON-LD data as a Ruby hash
+        # @param use_merge [Boolean] Use MERGE instead of CREATE
+        # @return [String] Cypher statement
+        def build(data, use_merge: false)
+          node_type = registry.node_type_for(data['@type'] || data['type'])
+          properties = mapper.map_properties(data)
+          node_id = generate_node_id(node_type, properties[:id])
+
+          if use_merge
+            build_merge_statement(node_id, node_type, properties)
+          else
+            build_create_statement(node_id, node_type, properties)
+          end
+        end
+
+        # Returns the property count for a given data hash
+        #
+        # @param data [Hash] The JSON-LD data
+        # @return [Integer] Number of properties
+        def property_count(data)
+          mapper.property_count(data)
+        end
+
+        private
+
+        # Builds a CREATE statement (always creates new node)
+        def build_create_statement(node_id, node_type, properties)
+          props_str = format_properties(properties)
+          "CREATE (#{node_id}:#{node_type} {#{props_str}});"
+        end
+
+        # Builds a MERGE statement (idempotent - creates only if doesn't exist)
+        def build_merge_statement(node_id, node_type, properties)
+          props_str = format_properties(properties)
+          <<~CYPHER.chomp
+            MERGE (#{node_id}:#{node_type} {id: #{properties[:id]}})
+            SET #{node_id} += {#{props_str}};
+          CYPHER
+        end
+
+        # Formats properties for Cypher
+        def format_properties(properties)
+          properties.map do |k, v|
+            "#{k}: #{v}"
+          end.join(', ')
+        end
+
+        # Generates a Neo4j-compatible node identifier
+        #
+        # @param node_type [String] The Neo4j label
+        # @param id [String] The unique identifier
+        # @return [String] Neo4j node identifier
+        def generate_node_id(node_type, id)
+          return "n_#{node_type.downcase}" unless id
+
+          # Create Neo4j-safe ID from the @id value
+          id.split('/').last
+            .gsub(/[^a-zA-Z0-9]/, '_')
+            .downcase
+            .then { |s| s.empty? ? "n_#{node_type.downcase}" : "#{s}_" }
+        end
+      end
+    end
+  end
+end

data/lib/openphar/exporters/neo4j/property_mapper.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Openphar
+  module Exporters
+    module Neo4j
+      # Maps model attributes to Neo4j-compatible property names
+      #
+      # Neo4j doesn't allow dots in property names, so this mapper
+      # converts camelCase JSON-LD keys to snake_case Neo4j properties.
+      #
+      # Uses the model registry to understand the structure of each model type.
+      class PropertyMapper
+        # Property name mapping: JSON-LD key => Neo4j property key
+        # Neo4j doesn't allow dots in property names, so we flatten
+        PROPERTY_MAPPING = {
+          '@id' => :id,
+          '@type' => :type,
+          'monographId' => :monograph_id,
+          'prefLabel' => :pref_label,
+          'altLabel' => :alt_label,
+          'definition' => :definition,
+          'publisher' => :publisher,
+          'status' => :status,
+          'belongsToEdition' => :edition_id,
+          'effectiveDate' => :effective_date,
+          'version' => :version,
+          'referencesPreparation' => :prep_id,
+          'referencesTCMProfile' => :tcm_id,
+          'referencesAyurvedaProfile' => :ayurveda_id,
+          'referencesWesternProfile' => :western_id,
+          'referencesPlantSpecies' => :plant_id,
+          'sameSubstanceAs' => :same_as,
+          'hasEquivalentIn' => :equiv_in,
+          'similarTo' => :similar_to,
+          'testSpecification' => :test_specs,
+          'storageContainer' => :storage_container,
+          'storageConditions' => :storage_conds,
+          'botanicalSource' => :botanical_source,
+          'macroscopicDescription' => :macro_desc,
+          'microscopicDescription' => :micro_desc,
+          'molecularFormula' => :mol_formula,
+          'molecularWeight' => :mol_weight,
+          'casNumber' => :cas_number,
+          'testName' => :test_name,
+          'testType' => :test_type,
+          'harmonizedMethod' => :harm_method,
+          'testConditions' => :test_conds,
+          'limitType' => :limit_type,
+          'limitValue' => :limit_value,
+          'lowerLimit' => :lower_limit,
+          'upperLimit' => :upper_limit,
+          'limitUnit' => :limit_unit,
+          'appearance' => :appearance,
+          'identification' => :identification,
+          'purity' => :purity,
+          'assay' => :assay,
+          'editionNumber' => :edition_number,
+          'publicationDate' => :publication_date,
+          'monographCount' => :monograph_count,
+          'officialLanguage' => :official_languages,
+          'supplementNumber' => :supplement_number,
+          # Ph.Int.-specific
+          'systematicName' => :systematic_name,
+          'inchi' => :inchi,
+          'inchiKey' => :inchi_key,
+          'smiles' => :smiles,
+          'hasRadionuclide' => :radionuclide,
+          'halfLife' => :half_life,
+          'emissionType' => :emission_type,
+          'radiochemicalPurity' => :radiochem_purity,
+          'radionuclidicPurity' => :radionucl_purity,
+          'photonFraction' => :photon_fraction,
+          'solubility' => :solubility,
+          'category' => :category,
+          'chemicalName' => :chemical_name
+        }.freeze
+
+        def initialize(registry = nil)
+          @registry = registry || ModelRegistry.new
+        end
+
+        # Maps a JSON-LD key to a Neo4j-compatible property key
+        #
+        # @param jsonld_key [String] The key from JSON-LD data
+        # @return [Symbol] Neo4j-compatible property key
+        def map_property(jsonld_key)
+          PROPERTY_MAPPING[jsonld_key] || default_mapping(jsonld_key)
+        end
+
+        # Maps all properties from JSON-LD data to Neo4j format
+        #
+        # @param data [Hash] The JSON-LD data as a Ruby hash
+        # @return [Hash] Mapped properties with Neo4j-compatible keys
+        def map_properties(data)
+          props = {}
+
+          data.each do |key, value|
+            neo4j_key = map_property(key)
+            props[neo4j_key] = convert_value(value)
+          end
+
+          props
+        end
+
+        # Converts a value to Neo4j-compatible format
+        #
+        # @param value [any] The value to convert
+        # @return [String] Neo4j-compatible value representation
+        def convert_value(value)
+          case value
+          when Hash then value.to_json
+          when Array then value.to_json
+          when String then escape_string(value)
+          when nil then 'null'
+          when Numeric then value.to_s
+          else value.to_s
+          end
+        end
+
+        # Returns the property count for analytics
+        #
+        # @param data [Hash] The JSON-LD data
+        # @return [Integer] Number of properties
+        def property_count(data)
+          data.keys.size
+        end
+
+        private
+
+        # Default mapping for unknown keys: camelCase to snake_case
+        def default_mapping(key)
+          key.gsub(/([^A-Z])([A-Z])/, '\1_\2')
+             .gsub(/[^a-zA-Z0-9]/, '_')
+             .downcase
+             .to_sym
+        end
+
+        # Escapes a string for Neo4j Cypher
+        def escape_string(str)
+          return 'null' if str.nil?
+          return str.to_s if str.is_a?(Numeric)
+
+          escaped = str.to_s
+                       .gsub('\\', '\\\\')
+                       .gsub("'", "\\'")
+                       .gsub("\n", '\\n')
+                       .gsub("\r", '\\r')
+
+          "'#{escaped}'"
+        end
+      end
+    end
+  end
+end