health-data-standards 2.2.1 → 3.0.3

data/lib/hqmf-parser/converter/pass2/operator_converter.rb

@@ -0,0 +1,102 @@
+module HQMF
+  # Class for converting an HQMF 1.0 representation to an HQMF 2.0 representation
+  class OperatorConverter
+    
+    def self.apply_temporal(data_criteria, precondition, restriction, data_criteria_converter)
+      data_criteria.temporal_references ||= []
+      value = restriction.operator.value
+      type = restriction.operator.type
+      temporal_reference = nil
+      if (restriction.single_target?)
+        # multiple targets appears to be the result of restrictions with restrictions
+        target = restriction.target
+        if (restriction.multi_target?)
+          found = false
+          # restrictions with restrictions can have a target that is modified by the child restrcitons
+          restriction.preconditions.each do |precondition|
+            if precondition.reference.id.start_with? target
+              found = true
+              target = precondition.reference.id
+            end
+          end
+          unless found
+            puts "\tmultiple targets... need to check this" if restriction.multi_target?
+          end
+        end
+        temporal_reference = HQMF::TemporalReference.new(type, HQMF::Reference.new(target),value)
+        data_criteria_converter.validate_not_deleted(target)
+      elsif (restriction.multi_target?)
+        
+        children_criteria = HQMF::DataCriteriaConverter.extract_data_criteria(restriction.preconditions, data_criteria_converter)
+        
+        if (children_criteria.length == 1)
+          target = children_criteria[0].id
+          temporal_reference = HQMF::TemporalReference.new(type, HQMF::Reference.new(target),value)
+          data_criteria_converter.validate_not_deleted(target)
+        else
+          parent_id = "GROUP"
+          if restriction.generated_data_criteria.nil?
+            # we pass in restriction.preconditions here rather than children_criteria because we need to be able to create grouping data criteria for and and or preconditions in a tree
+            group_criteria = data_criteria_converter.create_group_data_criteria(restriction.preconditions, "#{type}_CHILDREN", value, parent_id, HQMF::Counter.instance.next, "grouping", "temporal")
+            # save the generated grouping criteria so that we can reference it from other locations
+            restriction.generated_data_criteria = group_criteria
+          else
+            # we have already processed this restriction and have a grouping criteria for it.  Take the one we have previously generated
+            group_criteria = restriction.generated_data_criteria
+          end
+          temporal_reference = HQMF::TemporalReference.new(type, HQMF::Reference.new(group_criteria.id), value)
+        end
+      else
+        raise "no target for temporal restriction"
+      end
+      restriction.converted=true
+      
+      # add temporal reference to data criteria
+      data_criteria.temporal_references << temporal_reference unless data_criteria.has_temporal(temporal_reference)
+    end
+    
+    
+    def self.apply_summary(data_criteria, precondition, restriction, data_criteria_converter)
+      value = restriction.operator.value
+      type = restriction.operator.type
+      subset_operator = HQMF::SubsetOperator.new(type, value)
+
+      if (restriction.multi_target?)
+        children_criteria = HQMF::DataCriteriaConverter.extract_data_criteria(restriction.preconditions, data_criteria_converter)
+        
+        data_criteria = nil
+        if (children_criteria.length == 1)
+          data_criteria = children_criteria[0]
+          data_criteria.subset_operators ||= []
+          # add subset operator to data criteria
+          data_criteria.subset_operators << subset_operator unless data_criteria.has_subset(subset_operator)
+        else
+          parent_id = "GROUP"
+          
+          if restriction.generated_data_criteria.nil?
+            # we pass in restriction.preconditions here rather than children_criteria because we need to be able to create grouping data criteria for and and or preconditions in a tree
+            data_criteria = data_criteria_converter.create_group_data_criteria(restriction.preconditions, type, value, parent_id, HQMF::Counter.instance.next, "grouping", "summary")
+            # save the generated grouping criteria so that we can reference it from other locations
+            restriction.generated_data_criteria = data_criteria
+          else
+            # we have already processed this restriction and have a grouping criteria for it.  Take the one we have previously generated
+            data_criteria = restriction.generated_data_criteria
+          end
+          
+          data_criteria.subset_operators ||= []
+          # add subset operator to data criteria
+          data_criteria.subset_operators << subset_operator unless data_criteria.has_subset(subset_operator)
+        end
+        precondition.reference = HQMF::Reference.new(data_criteria.id)
+      elsif (restriction.single_target?)
+        subset_operator = HQMF::SubsetOperator.new(type, value)
+        data_criteria.subset_operators ||= []
+        # add subset operator to data criteria
+        data_criteria.subset_operators << subset_operator unless data_criteria.has_subset(subset_operator)
+      end
+      
+      restriction.converted=true
+    end
+    
+  end  
+end

data/lib/hqmf-parser/parser.rb

@@ -0,0 +1,54 @@
+module HQMF
+  class Parser
+    
+    HQMF_VERSION_1 = "1.0"
+    HQMF_VERSION_2 = "2.0"
+    
+    def self.parse(hqmf_contents, version, codes = nil)
+      
+      HQMF::Counter.instance.reset()
+      case version
+        when HQMF_VERSION_1
+          puts("\tCodes not passed in, cannot backfill properties like gender") unless codes
+          HQMF::DocumentConverter.convert(HQMF1::Document.new(hqmf_contents).to_json, codes)
+        when HQMF_VERSION_2
+          HQMF2::Document.new(hqmf_contents).to_model
+        else
+          raise "Unsupported HQMF version specified: #{version}"
+      end
+    end
+    
+    def self.parse_fields(hqmf_contents, version)
+      result = {}
+      case version
+        when HQMF_VERSION_1
+          doc = HQMF1::Document.parse(hqmf_contents)
+          type = doc.at_xpath('//cda:code/@code').value
+          if type == '57024-2'
+            id = doc.at_xpath('//cda:id/@root').value.upcase
+            set_id = doc.at_xpath('//cda:setId/@root').value.upcase
+            version_number = doc.at_xpath('//cda:versionNumber/@value').value.to_i
+            title = doc.at_xpath('cda:QualityMeasureDocument/cda:title').inner_text
+            description = doc.at_xpath('cda:QualityMeasureDocument/cda:text').inner_text
+            result = {'id' => id, 'set_id' => set_id, 'version' => version_number, 'title' => title, 'description' => description}
+          end
+        when HQMF_VERSION_2
+          doc = HQMF2::Document.parse(hqmf_contents)
+          type = doc.at_xpath('cda:QualityMeasureDocument/cda:code/@code').value
+          if type == '57024-2'
+            id = doc.at_xpath('cda:QualityMeasureDocument/cda:id/@extension', HQMF2::Document::NAMESPACES).value.upcase
+            set_id = doc.at_xpath('cda:QualityMeasureDocument/cda:setId/@extension').value.upcase
+            version_number = doc.at_xpath('cda:QualityMeasureDocument/cda:versionNumber/@value').value.to_i
+            title = doc.at_xpath('cda:QualityMeasureDocument/cda:title/@value').inner_text
+            description = doc.at_xpath('cda:QualityMeasureDocument/cda:text/@value').inner_text
+            result = {'id' => id, 'set_id' => set_id, 'version' => version_number, 'title' => title, 'description' => description}
+          end
+        else
+          raise "Unsupported HQMF version specified: #{version}"
+      end
+      result
+    end
+
+  end
+  
+end

data/lib/hqmf-parser/value_sets/value_set_parser.rb

@@ -0,0 +1,241 @@
+require 'zip/zipfilesystem'
+require 'spreadsheet'
+require 'google_spreadsheet'
+require 'roo'
+require 'iconv'
+
+module HQMF
+  module ValueSet
+    class Parser
+
+      attr_accessor :child_oids
+  
+      GROUP_CODE_SET = "GROUPING"
+  
+      ORGANIZATION_TITLE = "Value Set Developer"
+      OID_TITLE = "Value Set OID"
+      CONCEPT_TITLE = "Value Set Name"
+      CATEGORY_TITLE = "QDM Category"
+      CODE_SET_TITLE ="Code System"
+      VERSION_TITLE = "Code System Version"
+      CODE_TITLE = "Code"
+      DESCRIPTION_TITLE = "Descriptor"
+      
+      CODE_SYSTEM_NORMALIZER = {
+        'ICD-9'=>'ICD-9-CM',
+        'ICD-10'=>'ICD-10-CM',
+        'HL7 (2.16.840.1.113883.5.1)'=>'HL7'
+      }
+      IGNORED_CODE_SYSTEM_NAMES = ['Grouping', 'GROUPING' ,'HL7', "Administrative Sex"]
+  
+      def initialize()
+        @child_oids = []
+      end
+  
+      # import an excel matrix array into mongo
+      def parse(file, options={})
+        sheet_array = file_to_array(file, options)
+        by_oid_ungrouped = cells_to_hashs_by_oid(sheet_array)
+        value_sets = collapse_groups(by_oid_ungrouped)
+        translate_json(value_sets)
+      end
+  
+      def collapse_groups(by_oid_ungrouped)
+    
+        final = []
+    
+        # select the grouped code sets and fill in the children... also remove the children that are a
+        # member of a group.  We remove the children so that we can create parent groups for the orphans
+        (by_oid_ungrouped.select {|key,value| value["code_set"].upcase == GROUP_CODE_SET}).each do |key, value|
+          # remove the group so that it is not in the orphan list
+          by_oid_ungrouped.delete(value["oid"])
+          codes = []
+          value["codes"].each do |child_oid|
+#            codes << by_oid_ungrouped.delete(child_oid)
+            # do not delete the children of a group.  These may be referenced by other groups or directly by the measure
+            code = by_oid_ungrouped[child_oid]
+            @child_oids << child_oid
+            puts "\tcode could not be found: #{child_oid}" unless code
+            codes << code if code
+            # for hierarchies we need to probably have codes be a hash that we select from if we don't find the
+            # element in by_oid_ungrouped we may need to look for it in final
+          end
+          value["code_sets"] = codes
+          value.delete("codes")
+          value.delete("code_set")
+          final << value
+        end
+    
+        # fill out the orphans
+        by_oid_ungrouped.each do |key, orphan|
+          final << adopt_orphan(orphan)
+        end
+        
+        deleted = []
+        final.delete_if {|x| to_delete = x['code_sets'].nil? || x['code_sets'].empty?; deleted << x if to_delete; to_delete }
+        deleted.each do |value|
+          puts "\tDeleted value set with no code sets: #{value['oid']}"
+        end
+        final
+    
+      end
+  
+      def adopt_orphan(orphan)
+        parent = orphan.dup
+        parent["code_sets"] = [orphan]
+        parent.delete("codes")
+        parent.delete("code_set")
+        parent
+      end
+  
+      # take an excel matrix array and turn it into an array of db models
+      def cells_to_hashs_by_oid(array)
+        a = Array.new(array)                  # new variable for reentrant
+        headers = a.shift.map {|i| i.to_s }   # because of this shift
+        string_data = a.map {|row| row.map {|cell| cell.to_s } }
+        array_of_hashes = string_data.map {|row| Hash[*headers.zip(row).flatten] }
+
+        by_oid = {}
+        array_of_hashes.each do |row|
+          entry = convert_row(row)
+          
+          existing = by_oid[entry["oid"]]
+          if (existing)
+            existing["codes"].concat(entry["codes"])
+          else
+            by_oid[entry["oid"]] = entry
+          end
+        end
+    
+        by_oid
+      end
+  
+      def self.get_format(file_path)
+        if file_path =~ /xls$/
+          :xls
+        elsif file_path =~ /xlsx$/
+          :xlsx
+        end
+      end
+  
+      private
+  
+      def convert_row(row)
+        # Value Set Developer
+        # Value Set OID
+        # Value Set Name
+        # QDM Category
+        # Code System
+        # Code System Version
+        # Code
+        # Descriptor
+        value = {
+          "key" => normalize_names(row[CATEGORY_TITLE],row[CONCEPT_TITLE]),
+          "organization" => row[ORGANIZATION_TITLE],
+          "oid" => row[OID_TITLE].strip.gsub(/[^0-9\.]/i, ''),
+          "concept" => normalize_names(row[CONCEPT_TITLE]),
+          "category" => normalize_names(row[CATEGORY_TITLE]),
+          "code_set" => normalize_code_system(row[CODE_SET_TITLE]),
+          "version" => row[VERSION_TITLE],
+          "codes" => extract_code(row[CODE_TITLE], row[CODE_SET_TITLE]),
+          "description" => row[DESCRIPTION_TITLE]
+        }
+        value['codes'].map! {|code| code.strip.gsub(/[^0-9\.]/i, '')} if (value['code_set'].upcase == GROUP_CODE_SET)
+        value
+      end
+  
+      # Break all the supplied strings into separate words and return the resulting list as a
+      # new string with each word separated with '_'
+      def normalize_names(*components)
+        name = []
+        components.each do |component|
+          name.concat component.gsub(/\W/,' ').split.collect { |word| word.strip.downcase }
+        end
+        name.join '_'
+      end
+      
+      def normalize_code_system(code_system_name)
+        code_system_name = CODE_SYSTEM_NORMALIZER[code_system_name] if CODE_SYSTEM_NORMALIZER[code_system_name]
+        return code_system_name if IGNORED_CODE_SYSTEM_NAMES.include? code_system_name
+        oid = HealthDataStandards::Util::CodeSystemHelper.oid_for_code_system(code_system_name)
+        puts "\tbad code system name: #{code_system_name}" unless oid
+        code_system_name
+      end
+  
+      def extract_code(code, set)
+    
+        code.strip!
+        if set=='CPT' && code.include?('-')
+          eval(code.strip.gsub('-','..')).to_a.collect { |i| i.to_s }
+        else
+          [code]
+        end
+    
+      end
+  
+      def file_to_array(file_path, options)
+        defaults = {
+          :columns => 2,  # range of import
+          :sheet => 1     # only one sheet at a time can be worked on
+        }
+        options = defaults.merge(options)
+        
+        book = book_by_format(file_path, options[:format])
+        book.default_sheet=book.sheets[options[:sheet]]
+    
+        # catch double byte encoding problems in spreadsheet files
+        # Encoding::InvalidByteSequenceError: "\x9E\xDE" on UTF-16LE
+        begin
+          book.to_matrix.to_a
+        rescue Encoding::InvalidByteSequenceError => e
+          raise "Spreadsheet encoding problem: #{e}"
+        end
+      end
+      
+      def book_by_format(file_path, format)
+        format = HQMF::ValueSet::Parser.get_format(file_path) unless format
+        
+        if format == :xls
+          book = Excel.new(file_path, nil, :ignore)
+        elsif format == :xlsx
+          book = Excelx.new(file_path, nil, :ignore)
+        else
+          raise "File does not end in .xls or .xlsx"
+        end
+        book
+      end
+
+      def translate_json(value_sets)
+        value_set_models = []
+
+        value_sets.each do |value_set|
+          hds_value_set = HealthDataStandards::SVS::ValueSet.new() 
+          hds_value_set['oid'] = value_set['oid']
+          hds_value_set['display_name'] = value_set['key']
+          hds_value_set['version'] = value_set['version']
+          hds_value_set['concepts'] = []
+
+          value_set['code_sets'].each do |code_set|
+            code_set['codes'].map{ |code| 
+              concept = HealthDataStandards::SVS::Concept.new()
+              concept['code'] = code
+              concept['code_system'] = nil
+              concept['code_system_name'] = code_set['code_set']
+              concept['code_system_version'] = code_set['version']
+              concept['display_name'] = nil
+              hds_value_set['concepts'].concat([concept])
+            }
+          end
+          if hds_value_set['concepts'].include? nil
+            puts "Value Set has a bad code set (code set is null)"
+            hds_value_set['concepts'].compact!
+          end
+          value_set_models << hds_value_set
+        end
+        value_set_models
+      end
+  
+  
+    end
+  end
+end

data/lib/util/counter.rb

@@ -0,0 +1,20 @@
+module HQMF
+  # Simple class to issue monotonically increasing integer identifiers
+  class InstanceCounter
+    def initialize
+      @count=0
+    end
+
+    def reset
+      @count=0
+    end
+
+    def next
+      @count+=1
+    end
+  end
+
+  class Counter < InstanceCounter
+    include Singleton
+  end
+end