hqmf-parser 1.0.0

data/lib/hqmf-parser/converter/pass2/operator_converter.rb

@@ -0,0 +1,169 @@
+module HQMF
+  # Class for converting an HQMF 1.0 representation to an HQMF 2.0 representation
+  class OperatorConverter
+    
+    def self.apply_temporal(data_criteria, precondition, restriction, data_criteria_converter)
+      data_criteria.temporal_references ||= []
+      value = restriction.operator.value
+      type = restriction.operator.type
+      temporal_reference = nil
+      if (restriction.single_target?)
+        # multiple targets appears to be the result of restrictions with restrictions
+        target = restriction.target
+        if (restriction.multi_target?)
+          found = false
+          # restrictions with restrictions can have a target that is modified by the child restrcitons
+          restriction.preconditions.each do |precondition|
+            if precondition.reference.id.start_with? target
+              found = true
+              target = precondition.reference.id
+            end
+          end
+          unless found
+            puts "\tmultiple targets... need to check this" if restriction.multi_target?
+          end
+        end
+        temporal_reference = HQMF::TemporalReference.new(type, HQMF::Reference.new(target),value)
+        data_criteria_converter.validate_not_deleted(target)
+      elsif (restriction.multi_target?)
+        
+        children_criteria = HQMF::DataCriteriaConverter.extract_data_criteria(restriction.preconditions, data_criteria_converter)
+        
+        if (children_criteria.length == 1)
+          target = children_criteria[0].id
+          temporal_reference = HQMF::TemporalReference.new(type, HQMF::Reference.new(target),value)
+          data_criteria_converter.validate_not_deleted(target)
+        else
+          parent_id = "GROUP"
+          if restriction.generated_data_criteria.nil?
+            # we pass in restriction.preconditions here rather than children_criteria because we need to be able to create grouping data criteria for and and or preconditions in a tree
+            group_criteria = data_criteria_converter.create_group_data_criteria(restriction.preconditions, "#{type}_CHILDREN", value, parent_id, @@ids.next, "grouping", "temporal")
+            # save the generated grouping criteria so that we can reference it from other locations
+            restriction.generated_data_criteria = group_criteria
+          else
+            # we have already processed this restriction and have a grouping criteria for it.  Take the one we have previously generated
+            group_criteria = restriction.generated_data_criteria
+          end
+          temporal_reference = HQMF::TemporalReference.new(type, HQMF::Reference.new(group_criteria.id), value)
+        end
+      else
+        raise "no target for temporal restriction"
+      end
+      restriction.converted=true
+      
+      # add temporal reference to data criteria
+      data_criteria.temporal_references << temporal_reference unless data_criteria.has_temporal(temporal_reference)
+    end
+    
+    
+    def self.apply_summary(data_criteria, precondition, restriction, data_criteria_converter)
+      value = restriction.operator.value
+      type = restriction.operator.type
+      subset_operator = HQMF::SubsetOperator.new(type, value)
+
+      if (restriction.multi_target?)
+        children_criteria = HQMF::DataCriteriaConverter.extract_data_criteria(restriction.preconditions, data_criteria_converter)
+        
+        data_criteria = nil
+        if (children_criteria.length == 1)
+          data_criteria = children_criteria[0]
+          # this block used to set the order of operations of values after subset operators.
+          # if data_criteria and !data_criteria.value.nil?
+          #   subset_operator.value ||= data_criteria.value 
+          #   data_criteria.value = nil
+          # end
+          data_criteria.subset_operators ||= []
+          # add subset operator to data criteria
+          data_criteria.subset_operators << subset_operator unless data_criteria.has_subset(subset_operator)
+        else
+          parent_id = "GROUP"
+          
+          # this block of code used to be used to pull the value off of a data criteria and shove it onto a subset operator to evaluate MIN: lab result(result > 10) as MIN > 10: lab result
+          # the decision was made by NQF to not support this order of operations based on previous guidance to the measure developers
+          #
+          # unless subset_operator.value
+          #   # scalar comparisons are used for MIN>90 etc.  The value is on a REFR restriction.  We need to store it on the data criteria since the value is processed before the operator is created.
+          #   scalar_comparison = nil
+          #   
+          #   # check to see if we have different values referenced accross the children
+          #   # this happens on things like most recent blood pressure reading
+          #   multiple_differing_values = false
+          #   children_criteria.each do |criteria|
+          #     if scalar_comparison.nil?
+          #       scalar_comparison = criteria.value
+          #     else
+          #       if scalar_comparison != criteria.value
+          #         multiple_differing_values = true
+          #       end
+          #     end
+          #   end 
+          #   
+          #   # if we have multiple differing values, we want to keep the preconditions from the restriction, and attach them to the precondition
+          #   # we can then apply the subset operator to all of the children individually
+          #   if (multiple_differing_values)
+          #     precondition.preconditions.delete(restriction)
+          #     precondition.preconditions.concat(restriction.preconditions)
+          #     children_criteria.each do |criteria|
+          #       subset_operator = HQMF::SubsetOperator.new(type, value)
+          #       subset_operator.value = criteria.value
+          #       criteria.value = nil
+          #       criteria.subset_operators ||= []
+          #       criteria.subset_operators << subset_operator unless criteria.has_subset(subset_operator)
+          #     end
+          #     restriction.converted=true
+          #     # we want to return since we have applied the subset to the children.  We no longer want to create a grouping data critiera
+          #     return;
+          #   end
+          #   
+          #   # all the children have the same value, apply the value to the subset operator
+          #   children_criteria.each do |criteria|
+          #     if scalar_comparison.nil?
+          #       scalar_comparison = criteria.value
+          #     else
+          #       raise "multiple different scalar comparisons for a grouping data criteria" if scalar_comparison != criteria.value
+          #     end
+          #     criteria.value = nil
+          #   end 
+          #   subset_operator.value ||= scalar_comparison
+          # end
+          
+          if restriction.generated_data_criteria.nil?
+            # we pass in restriction.preconditions here rather than children_criteria because we need to be able to create grouping data criteria for and and or preconditions in a tree
+            data_criteria = data_criteria_converter.create_group_data_criteria(restriction.preconditions, type, value, parent_id, @@ids.next, "grouping", "summary")
+            # save the generated grouping criteria so that we can reference it from other locations
+            restriction.generated_data_criteria = data_criteria
+          else
+            # we have already processed this restriction and have a grouping criteria for it.  Take the one we have previously generated
+            data_criteria = restriction.generated_data_criteria
+          end
+          
+          data_criteria.subset_operators ||= []
+          # add subset operator to data criteria
+          data_criteria.subset_operators << subset_operator unless data_criteria.has_subset(subset_operator)
+        end
+        precondition.reference = HQMF::Reference.new(data_criteria.id)
+      elsif (restriction.single_target?)
+        subset_operator = HQMF::SubsetOperator.new(type, value)
+        data_criteria.subset_operators ||= []
+        # add subset operator to data criteria
+        data_criteria.subset_operators << subset_operator unless data_criteria.has_subset(subset_operator)
+      end
+      
+      restriction.converted=true
+    end
+    
+    # Simple class to issue monotonically increasing integer identifiers
+    class Counter
+      def initialize
+        @count = 0
+      end
+
+      def next
+        @count+=1
+      end
+    end
+    @@ids = Counter.new
+
+    
+  end  
+end

data/lib/hqmf-parser/converter/pass3/specific_occurrence_converter.rb

@@ -0,0 +1,86 @@
+module HQMF
+  # Class for cleaning up references to specific occurrences in the tree
+  class SpecificOccurrenceConverter
+    
+    def initialize(data_criteria_converter)
+      @data_criteria_converter = data_criteria_converter
+    end
+   
+    def convert_specific_occurrences(population_criteria)
+      population_criteria.each do |population|
+        walk_up_tree(population.preconditions)
+      end
+    end
+    
+    def walk_up_tree(preconditions)
+      preconditions.each do |precondition|
+        if (has_child_comparison(precondition))
+          walk_up_tree(precondition.preconditions)
+        end
+        unless (precondition.comparison?)
+          occurrence_map = get_grouped_occurrences(precondition.preconditions)
+          occurrence_map.each do |key, value|
+            
+            ##
+            ##### figure out negations
+            ##
+            
+            if (precondition.conjunction_code == 'atLeastOneTrue')
+              
+              group = @data_criteria_converter.build_group_data_criteria(value, "#{key}_union", precondition.id, HQMF::DataCriteria::UNION)
+              binding.pry
+            elsif (precondition.conjunction_code == 'allTrue')
+              binding.pry
+            end
+            
+          end
+        end
+      end
+    end
+    
+    def get_grouped_occurrences(preconditions)
+      
+      result = {}
+      preconditions.each do |precondition|
+        if (precondition.preconditions and !precondition.preconditions.empty?)
+          get_grouped_occurrences(precondition.preconditions).each do |key, value|
+            if (result[key])
+              result[key].concat(value)
+            else
+              result[key] = value
+            end
+          end
+        else
+          data_criteria = @data_criteria_converter.v2_data_criteria_by_id[precondition.reference.id]
+          if is_specific_occurrence(data_criteria)
+            binding.pry
+            result[data_criteria.source_data_criteria] ||= []
+            result[data_criteria.source_data_criteria] << data_criteria
+            preconditions.delete(precondition)
+            binding.pry
+          end
+        end
+      end
+      result
+      
+    end
+    
+    def is_specific_occurrence(data_criteria)
+      !data_criteria.specific_occurrence.nil?
+    end
+    
+    def has_child_comparison(node)
+      value = false
+      node.preconditions.each do |precondition|
+        if (precondition.comparison?)
+          value ||= true
+        elsif precondition.has_preconditions?
+          value ||= has_child_comparison(precondition)
+        end
+      end if node.preconditions
+      value
+    end
+    
+    
+  end  
+end

data/lib/hqmf-parser/converter/pass3/specific_occurrence_converter_bak.rb

@@ -0,0 +1,70 @@
+module HQMF
+  # Class for cleaning up references to specific occurrences in the tree
+  class SpecificOccurrenceConverter
+    
+    def initialize(data_criteria_converter)
+      @data_criteria_converter = data_criteria_converter
+    end
+   
+   def convert_specific_occurrences(population_criteria)
+     create_precondition_map(population_criteria)
+     
+     @specific_occurrence_key_mapper = {}
+     
+     specific_occurrences = {}
+     @data_criteria_converter.v2_data_criteria.each do |data_criteria|
+       if (is_specific_occurrence(data_criteria))
+         if (data_criteria.respond_to? :precondition_id and @precondition_map[data_criteria.precondition_id])
+           specific_occurrences[data_criteria.source_data_criteria] ||= []
+           specific_occurrences[data_criteria.source_data_criteria] << data_criteria
+         elsif (data_criteria.children_criteria or data_criteria.subset_operators or data_criteria.temporal_references)
+           binding.pry
+         end
+       end
+     end
+     specific_occurrences.each do |key, occurrences|
+       
+       occurrences.sort! do |left, right|
+         @precondition_map[right.precondition_id].depth <=> @precondition_map[left.precondition_id].depth
+       end
+       
+       occurrences.each_cons(2) do |pair|
+         parent = get_common_parent(@precondition_map[pair[0].precondition_id], @precondition_map[pair[1].precondition_id])
+         binding.pry
+       end
+       
+     end
+   end
+   
+   def get_common_parent(left, right)
+     return left if left == right
+     get_common_parent(@precondition_parent_map[left.id], @precondition_parent_map[right.id])
+   end
+   
+   def create_precondition_map(population_criteria)
+     @precondition_map = {}
+     @precondition_parent_map = {}
+     depth = 0
+     population_criteria.each do |population|
+       recurse_precondition_map(population, depth)
+     end
+   end
+   
+   def recurse_precondition_map(node, depth)
+     node.preconditions.each do |precondition|
+       precondition.depth = depth
+       @precondition_map[precondition.id] = precondition
+       @precondition_parent_map[precondition.id] = node
+       if precondition.has_preconditions?
+         depth += 1
+         recurse_precondition_map(precondition, depth)
+       end
+     end if node.preconditions
+   end
+   
+   def is_specific_occurrence(data_criteria)
+     !data_criteria.specific_occurrence.nil?
+   end
+   
+  end  
+end

data/lib/hqmf-parser/parser.rb

@@ -0,0 +1,22 @@
+module HQMF
+  class Parser
+    
+    HQMF_VERSION_1 = "1.0"
+    HQMF_VERSION_2 = "2.0"
+    
+    def self.parse(hqmf_contents, version, codes = nil)
+      
+      
+      case version
+        when HQMF_VERSION_1
+          puts("\tCodes not passed in, cannot backfill properties like gender") unless codes
+          HQMF::DocumentConverter.convert(HQMF1::Document.new(hqmf_contents).to_json, codes)
+        when HQMF_VERSION_2
+          HQMF2::Document.new(hqmf_contents).to_model
+        else
+          raise "Unsupported HQMF version specified: #{version}"
+        end
+      end
+    
+  end
+end

data/lib/hqmf-parser/value_sets/value_set_parser.rb

@@ -0,0 +1,206 @@
+require 'zip/zipfilesystem'
+require 'spreadsheet'
+require 'google_spreadsheet'
+require 'roo'
+require 'iconv'
+
+module HQMF
+  module ValueSet
+    class Parser
+  
+      GROUP_CODE_SET = "GROUPING"
+  
+      ORGANIZATION_TITLE = "Value Set Developer"
+      OID_TITLE = "Value Set OID"
+      CONCEPT_TITLE = "Value Set Name"
+      CATEGORY_TITLE = "QDM Category"
+      CODE_SET_TITLE ="Code System"
+      VERSION_TITLE = "Code System Version"
+      CODE_TITLE = "Code"
+      DESCRIPTION_TITLE = "Descriptor"
+      
+      CODE_SYSTEM_NORMALIZER = {
+        'ICD-9'=>'ICD-9-CM',
+        'ICD-10'=>'ICD-10-CM',
+        'HL7 (2.16.840.1.113883.5.1)'=>'HL7'
+      }
+      IGNORED_CODE_SYSTEM_NAMES = ['Grouping', 'GROUPING' ,'HL7', "Administrative Sex"]
+  
+      def initialize()
+      end
+  
+      # import an excel matrix array into mongo
+      def parse(file, options={})
+        sheet_array = file_to_array(file, options)
+        by_oid_ungrouped = cells_to_hashs_by_oid(sheet_array)
+        collapse_groups(by_oid_ungrouped)
+      end
+  
+      def collapse_groups(by_oid_ungrouped)
+    
+        final = []
+    
+        # select the grouped code sets and fill in the children... also remove the children that are a
+        # member of a group.  We remove the children so that we can create parent groups for the orphans
+        (by_oid_ungrouped.select {|key,value| value["code_set"].upcase == GROUP_CODE_SET}).each do |key, value|
+          # remove the group so that it is not in the orphan list
+          by_oid_ungrouped.delete(value["oid"])
+          codes = []
+          value["codes"].each do |child_oid|
+#            codes << by_oid_ungrouped.delete(child_oid)
+            # do not delete the children of a group.  These may be referenced by other groups or directly by the measure
+            code = by_oid_ungrouped[child_oid]
+            puts "\tcode could not be found: #{child_oid}" unless code
+            codes << code if code
+            # for hierarchies we need to probably have codes be a hash that we select from if we don't find the
+            # element in by_oid_ungrouped we may need to look for it in final
+          end
+          value["code_sets"] = codes
+          value.delete("codes")
+          value.delete("code_set")
+          final << value
+        end
+    
+        # fill out the orphans
+        by_oid_ungrouped.each do |key, orphan|
+          final << adopt_orphan(orphan)
+        end
+        
+        deleted = []
+        final.delete_if {|x| to_delete = x['code_sets'].nil? || x['code_sets'].empty?; deleted << x if to_delete; to_delete }
+        deleted.each do |value|
+          puts "\tDeleted value set with no code sets: #{value['oid']}"
+        end
+        final
+    
+      end
+  
+      def adopt_orphan(orphan)
+        parent = orphan.dup
+        parent["code_sets"] = [orphan]
+        parent.delete("codes")
+        parent.delete("code_set")
+        parent
+      end
+  
+      # take an excel matrix array and turn it into an array of db models
+      def cells_to_hashs_by_oid(array)
+        a = Array.new(array)                  # new variable for reentrant
+        headers = a.shift.map {|i| i.to_s }   # because of this shift
+        string_data = a.map {|row| row.map {|cell| cell.to_s } }
+        array_of_hashes = string_data.map {|row| Hash[*headers.zip(row).flatten] }
+
+        by_oid = {}
+        array_of_hashes.each do |row|
+          entry = convert_row(row)
+          
+          existing = by_oid[entry["oid"]]
+          if (existing)
+            existing["codes"].concat(entry["codes"])
+          else
+            by_oid[entry["oid"]] = entry
+          end
+        end
+    
+        by_oid
+      end
+  
+      def self.get_format(file_path)
+        if file_path =~ /xls$/
+          :xls
+        elsif file_path =~ /xlsx$/
+          :xlsx
+        end
+      end
+  
+      private
+  
+      def convert_row(row)
+        # Value Set Developer
+        # Value Set OID
+        # Value Set Name
+        # QDM Category
+        # Code System
+        # Code System Version
+        # Code
+        # Descriptor
+        value = {
+          "key" => normalize_names(row[CATEGORY_TITLE],row[CONCEPT_TITLE]),
+          "organization" => row[ORGANIZATION_TITLE],
+          "oid" => row[OID_TITLE].strip.gsub(/[^0-9\.]/i, ''),
+          "concept" => normalize_names(row[CONCEPT_TITLE]),
+          "category" => normalize_names(row[CATEGORY_TITLE]),
+          "code_set" => normalize_code_system(row[CODE_SET_TITLE]),
+          "version" => row[VERSION_TITLE],
+          "codes" => extract_code(row[CODE_TITLE], row[CODE_SET_TITLE]),
+          "description" => row[DESCRIPTION_TITLE]
+        }
+        value['codes'].map! {|code| code.strip.gsub(/[^0-9\.]/i, '')} if (value['code_set'].upcase == GROUP_CODE_SET)
+        value
+      end
+  
+      # Break all the supplied strings into separate words and return the resulting list as a
+      # new string with each word separated with '_'
+      def normalize_names(*components)
+        name = []
+        components.each do |component|
+          name.concat component.gsub(/\W/,' ').split.collect { |word| word.strip.downcase }
+        end
+        name.join '_'
+      end
+      
+      def normalize_code_system(code_system_name)
+        code_system_name = CODE_SYSTEM_NORMALIZER[code_system_name] if CODE_SYSTEM_NORMALIZER[code_system_name]
+        return code_system_name if IGNORED_CODE_SYSTEM_NAMES.include? code_system_name
+        oid = HealthDataStandards::Util::CodeSystemHelper.oid_for_code_system(code_system_name)
+        puts "\tbad code system name: #{code_system_name}" unless oid
+        code_system_name
+      end
+  
+      def extract_code(code, set)
+    
+        code.strip!
+        if set=='CPT' && code.include?('-')
+          eval(code.strip.gsub('-','..')).to_a.collect { |i| i.to_s }
+        else
+          [code]
+        end
+    
+      end
+  
+      def file_to_array(file_path, options)
+        defaults = {
+          :columns => 2,  # range of import
+          :sheet => 1     # only one sheet at a time can be worked on
+        }
+        options = defaults.merge(options)
+        
+        book = book_by_format(file_path, options[:format])
+        book.default_sheet=book.sheets[options[:sheet]]
+    
+        # catch double byte encoding problems in spreadsheet files
+        # Encoding::InvalidByteSequenceError: "\x9E\xDE" on UTF-16LE
+        begin
+          book.to_matrix.to_a
+        rescue Encoding::InvalidByteSequenceError => e
+          raise "Spreadsheet encoding problem: #{e}"
+        end
+      end
+      
+      def book_by_format(file_path, format)
+        format = HQMF::ValueSet::Parser.get_format(file_path) unless format
+        
+        if format == :xls
+          book = Excel.new(file_path, nil, :ignore)
+        elsif format == :xlsx
+          book = Excelx.new(file_path, nil, :ignore)
+        else
+          raise "File does not end in .xls or .xlsx"
+        end
+        book
+      end
+  
+  
+    end
+  end
+end