health-data-standards 3.4.5 → 3.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68d3be05b049b75724e22b58c9a19635903b9409
-  data.tar.gz: 42ae8b42651d913d9122a4591a6e6789b381ffaa
+  metadata.gz: fac1ae0fcbbd6b03dfdfe42628158c70acdca202
+  data.tar.gz: 80d25a9fdca4418440341a210a1ecad16bd12260
 SHA512:
-  metadata.gz: 30efed49b0060bba83cc88b4d1eaad0f93bd3fddc01d084d6330bf25884d61c2f0f3b850cabeb153df1a506e7f724d61afd28a6432015a2b826db0abb2181aeb
-  data.tar.gz: 5c500ccd6d7687fdcdfc211b1fbf4d9a47bc8148ff9737f302e5c710dcdf042082312aceeb7f5f55583aa964b619ecd0d0f81fed1bdccdf31db7d3a256de1976
+  metadata.gz: d6ce76c141974dd7d0b97c1b2603ced0aa6fa51f2bb5a0371111e7f3fd64ebb2377a6e0097953a509786401337b311a245fbd012835982798536c70823f6b734
+  data.tar.gz: ae3f4f96b405d1f10c4efb3411a583f70da9b8ac83e69b0ecdeffa3f02b0331de8868291f029296f771d1d62141eb68374a4726e8d8d650ee7271edf9d7e378a

data/README.md

@@ -1,6 +1,6 @@
 This is a project to generate and consume HITSP C32, ASTM CCR, QRDA Category I, QRDA Category III and PQRI.
 
-In addition this project also contains libaries for parsing HQMF documents and for dealing with NLM valuesets.
+In addition this project also contains libraries for parsing HQMF documents and for dealing with NLM valuesets.
 
 Environment
 ===========
@@ -25,9 +25,19 @@ Please try to follow the [GitHub Coding Style Guides](https://github.com/stylegu
 Change Log
 ==========
 
-3.4.5 - Not yet released
+3.4.6 - May 15, 2014
 
-* Performance improvents in all exports through template caching
+* QRDA Cat I export now exports medical record number if present
+* Measures can now generate prefilter queries to be passed to MongoDB before CQM MapReduce jobs
+* BulkRecordImporter now handles JSON records and has better error handling
+* Bug fix - QRDA Cat III export uses correct XML element name for representedCustodianOrganization
+* Bug fix - QRDA Cat III export now properly represents CV measures
+* Bug fix - QRDA Cat I import now properly stores procedure performed ordinality
+* Bug fix - InsuranceProvider model included ThingsWithCodes twice
+
+3.4.5 - April 4, 2014
+
+* Performance improvements in all exports through template caching
 * QRDA Cat I export now exports the record's actual address if present
 * QRDA Cat I export - Bug fix - previously patients with a race but no ethnicity would cause exceptions
 * QRDA Cat I import - performance improvements through more efficient XPath expressions
@@ -47,11 +57,11 @@ Change Log
 3.4.2 - February 28, 2014
 
 * Provider improvements
-** Providers are now exported in QRDA Cat I if they exist on the Record object
-** Provider identifiers are now represented with CDAIdentifiers
-** Provider import will now import any type of identifier
-** Providers can now be arranged in a hierarchy
-** When importing providers from a clinical document, matching to existing providers in MongoDB has been improved to matching on any type of identifier
+  * Providers are now exported in QRDA Cat I if they exist on the Record object
+  * Provider identifiers are now represented with CDAIdentifiers
+  * Provider import will now import any type of identifier
+  * Providers can now be arranged in a hierarchy
+  * When importing providers from a clinical document, matching to existing providers in MongoDB has been improved to matching on any type of identifier
 * QRDA Cat I importer now imports procedure values regardless of whether the template has them at the root level or nested in an entryRelationship
 * Bug fix - Result importer will no longer double import result values
 * QRDA Cat I importer now imports encounter order end times

data/lib/health-data-standards.rb

@@ -53,7 +53,7 @@ require_relative 'health-data-standards/models/medication'
 require_relative 'health-data-standards/models/procedure'
 require_relative 'health-data-standards/models/lab_result'
 require_relative 'health-data-standards/models/functional_status'
-require_relative 'health-data-standards/models/medical_equipment'  
+require_relative 'health-data-standards/models/medical_equipment'
 require_relative 'health-data-standards/models/record'
 require_relative 'health-data-standards/models/personable'
 require_relative 'health-data-standards/models/provider'
@@ -88,6 +88,7 @@ require_relative 'health-data-standards/models/qrda/header'
 require_relative 'health-data-standards/models/cqm/aggregate_objects'
 require_relative 'health-data-standards/models/cqm/query_cache'
 require_relative 'health-data-standards/models/cqm/bundle'
+require_relative 'health-data-standards/models/cqm/prefilter'
 require_relative 'health-data-standards/models/cqm/measure'
 require_relative 'health-data-standards/models/cqm/patient_cache'
 
@@ -149,7 +150,7 @@ require_relative 'health-data-standards/import/green_c32/encounter_importer'
 require_relative 'health-data-standards/import/green_c32/medication_importer'
 require_relative 'health-data-standards/import/green_c32/allergy_importer'
 require_relative 'health-data-standards/import/green_c32/social_history_importer'
-require_relative 'health-data-standards/import/green_c32/immunization_importer' 
+require_relative 'health-data-standards/import/green_c32/immunization_importer'
 require_relative 'health-data-standards/import/green_c32/support_importer'
 require_relative 'health-data-standards/import/green_c32/advance_directive_importer'
 require_relative 'health-data-standards/import/green_c32/medical_equipment_importer'
@@ -186,7 +187,7 @@ module HealthDataStandards
 end
 
 if defined?(Rails)
-  require_relative 'health-data-standards/railtie' 
+  require_relative 'health-data-standards/railtie'
 else
   HealthDataStandards.logger = Log4r::Logger.new("Health Data Standards")
   HealthDataStandards.logger.outputters = Log4r::Outputter.stdout

data/lib/health-data-standards/import/bulk_record_importer.rb

@@ -3,48 +3,59 @@ module HealthDataStandards
   module Import
     class BulkRecordImporter
 
-      def self.import_directory(source_dir)
-
-        xml_files = Dir.glob(File.join(source_dir, '*.*'))
-        xml_files.each do |file|
-          begin
-            
-            result = RecordImporter.import(File.new(file).read, provider_map)
-            
-            if (result[:status] == 'success') 
-              record = result[:record]
-              record.save
-            else 
-              assert result[:message]
-            end
-            
-          rescue Exception => e
-            failed_dir = File.join(source_dir, '../', 'failed_imports')
-            unless(Dir.exists?(failed_dir))
-              Dir.mkdir(failed_dir)
-            end
-            FileUtils.cp(file, failed_dir)
-          end
+      def self.import_directory(source_dir, failed_dir=nil)
+        failed_dir ||= File.join(source_dir, '../', 'failed_imports')
+        files = Dir.glob(File.join(source_dir, '*.*'))
+        files.each do |file|
+           BulkRecordImporter.import_file(file,File.new(file).read,failed_dir)
         end
       end
       
-      def self.import_archive(file) 
+      def self.import_archive(file, failed_dir=nil) 
+        begin 
+        failed_dir ||=File.join(File.dirname(file))
         Zip::ZipFile.open(file.path) do |zipfile|
           zipfile.entries.each do |entry|
             next if entry.directory?
-            xml = zipfile.read(entry.name)
-            begin
-              BulkRecordImporter.import(xml)
-            rescue Exception => e
-              failed_dir = File.join(file.dirname, 'failed_imports')
-              unless(Dir.exists?(failed_dir))
-                Dir.mkdir(failed_dir)
-              end
-              FileUtils.cp(file, failed_dir)
-            end
+            data = zipfile.read(entry.name)
+            BulkRecordImporter.import_file(entry.name,data,failed_dir)
+          end
+        end
+      rescue
+        FileUtils.mkdir_p(failed_dir)
+        File.cp(file,File.join(failed_dir,file))
+        File.open(File.join(failed_dir,"#{file}.error")) do |f|
+          f.puts($!.message)
+          f.puts($!.backtrace)
+        end
+        raise $!
+      end
+      end
+
+      def self.import_file(name,data,failed_dir,provider_map={})
+        begin
+          ext = File.extname(name)
+          if ext == ".json"
+            BulkRecordImporter.import_json(data)
+          else
+            BulkRecordImporter.import(data)
+          end
+        rescue
+          FileUtils.mkdir_p(File.dirname(File.join(failed_dir,name)))
+          File.open(File.join(failed_dir,name),"w") do |f|
+            f.puts(data)
+          end
+          File.open(File.join(failed_dir,"#{name}.error"),"w") do |f|
+            f.puts($!.message)
+            f.puts($!.backtrace)
           end
         end
       end
+
+      def self.import_json(data,provider_map = {})
+        json = JSON.parse(data,:max_nesting=>100)
+        Record.update_or_create(Record.new(json))
+      end
       
       def self.import(xml_data, provider_map = {})
         doc = Nokogiri::XML(xml_data)
@@ -84,8 +95,6 @@ module HealthDataStandards
           end
         end
         record.save
-        
-        {status: 'success', message: 'patient imported', status_code: 201, record: record}
 
       end
     end

data/lib/health-data-standards/import/cat1/procedure_performed_importer.rb

@@ -1,31 +1,20 @@
 module HealthDataStandards
   module Import
     module Cat1
-      class ProcedurePerformedImporter < CDA::SectionImporter
+      class ProcedurePerformedImporter < CDA::ProcedureImporter
         def initialize(entry_finder=CDA::EntryFinder.new("./cda:entry/cda:procedure[cda:templateId/@root = '2.16.840.1.113883.10.20.24.3.64']"))
           super(entry_finder)
-          @entry_class = Procedure
-          @ordinality_xpath = "./cda:priorityCode"
           @incision_xpath ="./cda:entryRelationship/cda:procedure[./cda:templateId[@root='2.16.840.1.113883.10.20.24.3.89']]/cda:effectiveTime"
         end
-        
+
         def create_entry(entry_element, nrh = CDA::NarrativeReferenceHandler.new)
           procedure = super
-          extract_ordinality(entry_element, procedure)
           extract_incision_date_time(entry_element,procedure)
-          extract_negation(entry_element, procedure)
           procedure
         end
 
         private
 
-        def extract_ordinality(parent_element, procedure)
-          ordinality_element = parent_element.at_xpath(@ordinality_xpath)
-          if ordinality_element
-            procedure.ordinality = {CodeSystemHelper.code_system_for(ordinality_element['codeSystem']) => [ordinality_element['code']]}
-          end
-        end
-
         def extract_incision_date_time(parent_element, procedure)
           incision_time = parent_element.at_xpath(@incision_xpath)
           if incision_time

data/lib/health-data-standards/models/cqm/measure.rb

@@ -29,6 +29,9 @@ module HealthDataStandards
       field :populations, type: Array
       field :preconditions, type: Hash
       field :hqmf_document, type: Hash
+
+      embeds_many :prefilters
+
       scope :top_level_by_type , ->(type){where({"type"=> type}).any_of({"sub_id" => nil}, {"sub_id" => "a"})}
       scope :top_level , any_of({"sub_id" => nil}, {"sub_id" => "a"})
       scope :order_by_id_sub_id, order_by([["id", :asc],["sub_id", :asc]])
@@ -135,6 +138,79 @@ module HealthDataStandards
         end
         @crit
       end
+
+      # Builds the query hash to pass to MongoDB
+      # Calling this method will create Prefilters if they do not exist on the
+      # measure
+      def prefilter_query!(effective_time)
+        self.build_pre_filters! if self.prefilters.empty?
+
+        if self.prefilters.count == 1
+          self.prefilters.first.build_query_hash(effective_time)
+        else
+          self.prefilters.inject({}) do |query, pf|
+            query.merge(pf.build_query_hash(effective_time)) do |key, new_val, old_val|
+              new_val.merge(old_val)
+            end
+          end
+        end
+      end
+
+      # For submeasures, this will return something like IPP_1
+      def ipp_id
+        ipp_hqmf_id = self.population_ids['IPP']
+        pop_id, pop_criteria = hqmf_document['population_criteria'].find do |population_id, population_criteria|
+          population_criteria['hqmf_id'] == ipp_hqmf_id
+        end
+        pop_id
+      end
+
+      def build_pre_filters!
+        dc = self.data_criteria.inject({}) do |all_dc, single_dc|
+          key = single_dc.keys.first
+          value = single_dc.values.first
+          all_dc[key] = value
+          all_dc
+        end
+        dc.each_pair do |criteria_name, data_criteria|
+          if data_criteria['definition'] == 'patient_characteristic_birthdate'
+            if data_criteria_in_population?(self.ipp_id, criteria_name)
+              prefilter = Prefilter.new(record_field: 'birthdate',
+                                        effective_time_based: true)
+              if data_criteria['temporal_references']
+                data_criteria['temporal_references'].each do |tr|
+                  if tr['type'] == 'SBS' && tr['reference'] == 'MeasurePeriod'
+                    years = nil
+                    if tr['range']['high']
+                      prefilter.comparison = '$gte'
+                      years = tr['range']['high']['value'].to_i
+                    elsif tr['range']['low']
+                      prefilter.comparison = '$lte'
+                      years = tr['range']['low']['value'].to_i
+                    end
+
+                    prefilter.effective_time_offset = 1 + years
+                    self.prefilters << prefilter
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+
+      private
+
+      def data_criteria_in_population?(population_id, criteria_name)
+        criteria_in_precondition?(self.hqmf_document['population_criteria'][population_id]['preconditions'], criteria_name)
+      end
+
+      def criteria_in_precondition?(preconditions, criteria_name)
+        preconditions.any? do |precondition|
+          (precondition['reference'] == criteria_name) ||
+          (precondition['preconditions'] && criteria_in_precondition?(precondition['preconditions'], criteria_name))
+        end
+      end
     end
   end
-end
+end

data/lib/health-data-standards/models/cqm/prefilter.rb

@@ -0,0 +1,34 @@
+module HealthDataStandards
+  module CQM
+    class Prefilter
+      include Mongoid::Document
+      # What field on the Record do we want to compare against
+      field :record_field, type: String
+      # greater than, less than, etc.
+      field :comparison, type: String
+      # Is it based on the effective_time, like 65yo during measure period
+      field :effective_time_based, type: Boolean, default: false
+      # If effective_time_based, what is the offset from effective_time in years
+      field :effective_time_offset, type: Integer
+      # Comparison to a plain old value, like gender == 'F'
+      field :desired_value
+
+      embedded_in :measure
+
+      def build_query_hash(effective_time)
+        filter_value = if self.effective_time_based
+          et = Time.at(effective_time)
+          et.years_ago(effective_time_offset).to_i
+        else
+          self.desired_value
+        end
+
+        if self.comparison == '$eq'
+          {self.record_field => desired_value}
+        else
+          {self.record_field => {self.comparison => filter_value}}
+        end
+      end
+    end
+  end
+end

data/lib/health-data-standards/models/insurance_provider.rb

@@ -1,6 +1,5 @@
 class InsuranceProvider < Entry
   include Mongoid::Document
-  include ThingWithCodes
   
   embedded_in :record, class_name: 'Record'
   embeds_one :payer, class_name: "Organization"

data/lib/health-data-standards/models/record.rb

@@ -2,7 +2,7 @@ class Record
   include Mongoid::Document
   include Mongoid::Timestamps
   extend Memoist
-  
+
   field :title, type: String
   field :first, type: String
   field :last, type: String
@@ -17,12 +17,13 @@ class Record
   field :test_id, type: Moped::BSON::ObjectId
   field :marital_status, type: Hash
   field :medical_record_number, type: String
+  field :medical_record_assigner, type: String
   field :expired, type: Boolean
   field :clinicalTrialParticipant, type: Boolean   # Currently not implemented in the C32 importer
                                                    # because it cannot be easily represented in a
                                                    # HITSP C32
 
-  index "last" => 1                                                   
+  index "last" => 1
   embeds_many :allergies
   embeds_many :care_goals, class_name: "Entry" # This can be any number of different entry types
   embeds_many :conditions
@@ -50,7 +51,7 @@ class Record
   embeds_many :provider_performances
   embeds_many :addresses, as: :locatable
   embeds_many :telecoms, as: :contactable
-  
+
   scope :by_provider, ->(prov, effective_date) { (effective_date) ? where(provider_queries(prov.id, effective_date)) : where('provider_performances.provider_id'=>prov.id)  }
   scope :by_patient_id, ->(id) { where(:medical_record_number => id) }
 
@@ -68,7 +69,7 @@ class Record
   def providers
     provider_performances.map {|pp| pp.provider }
   end
-  
+
   def over_18?
     Time.at(birthdate) < Time.now.years_ago(18)
   end
@@ -92,7 +93,7 @@ class Record
   end
 
   memoize :entries_for_oid
-  
+
   alias :clinical_trial_participant :clinicalTrialParticipant
   alias :clinical_trial_participant= :clinicalTrialParticipant=
 
@@ -139,16 +140,16 @@ class Record
 
   end
 
-  private 
-  
+  private
+
   def self.provider_queries(provider_id, effective_date)
    {'$or' => [provider_query(provider_id, effective_date,effective_date), provider_query(provider_id, nil,effective_date), provider_query(provider_id, effective_date,nil)]}
   end
   def self.provider_query(provider_id, start_before, end_after)
     {'provider_performances' => {'$elemMatch' => {'provider_id' => provider_id, '$and'=>[{'$or'=>[{'start_date'=>nil},{'start_date'=>{'$lt'=>start_before}}]}, {'$or'=>[{'end_date'=>nil},{'end_date'=> {'$gt'=>end_after}}]}] } }}
   end
-  
 
-  
+
+
 
 end

data/lib/health-data-standards/models/thing_with_codes.rb

@@ -24,14 +24,16 @@ module ThingWithCodes
     if matching_code_sets.present?
       if value_set_map
         matching_code_sets.each do |matching_code_set|
-           matching_codes = codes_value[matching_code_set] & value_set_map.collect{|cs| cs["set"] == matching_code_set ? cs["values"] : []}.flatten.compact
-           if matching_codes.present?
-             return {'code' => matching_codes.first, 'code_set' => matching_code_set}
-           end
+          matching_codes = codes_value[matching_code_set] & value_set_map.collect{|cs| cs["set"] == matching_code_set ? cs["values"] : []}.flatten.compact
+          if matching_codes.present?
+            return {'code' => matching_codes.first, 'code_set' => matching_code_set}
+          end
         end
+        # we did not find a matching preferred code... we cannot write this out to QRDA
+        return nil
       else
-      code_set = matching_code_sets.first
-      {'code' => codes_value[code_set].first, 'code_set' => code_set}
+        code_set = matching_code_sets.first
+        {'code' => codes_value[code_set].first, 'code_set' => code_set}
       end
     else
       nil

data/templates/cat1/_record_target.cat1.erb

@@ -1,13 +1,10 @@
   <recordTarget>
     <patientRole>
-      <!-- id root="Cypress" extension="<%= patient.id %>"/ -->
-      <!-- Fake Medicare HIC number -->
-      <%
-      # get the pateint id , treate it as a hex number, turn it to a string and strip the last 9 characters off of it
-        hic = patient.id.to_s.hex.to_s
-        hic = hic.slice(hic.length-9,9)
-      %>
-      <id extension="<%= hic %>A" root="2.16.840.1.113883.4.572"/>
+      <% if (patient.medical_record_number && patient.medical_record_assigner) %>
+        <id extension="<%= patient.medical_record_number %>" root="<%= patient.medical_record_assigner %>"/>
+      <% else -%>
+        <id extension="12345" root="PlaceholderOrganization" />
+      <% end -%>
       <addr use="HP">
         <% if patient.addresses.present?
             patient.addresses.each do |address| -%>

data/templates/cat3/_author.cat3.erb

@@ -22,7 +22,7 @@
        </assignedAuthoringDevice>
       <% end %> 
      
-     <%== render :partial=>"organization", :locals=>{organization: author.organization} %>
+     <%== render :partial=>"organization", :locals=>{organization: author.organization, tag_name: "representedOrganization"} %>
 
     </assignedAuthor>
   </author>

data/templates/cat3/_continuous_variable_value.cat3.erb

@@ -5,7 +5,7 @@
       <originalText>Time Difference</originalText>
     </code>
     <statusCode code="completed"/>
-    <value xsi:type="PQ" value="<%= population.value %>" unit="min"/>
+    <value xsi:type="PQ" value="<%= value %>" unit="min"/>
     <methodCode code="MEDIAN" 
                 displayName="Median" 
                 codeSystem="2.16.840.1.113883.5.84" 
@@ -13,7 +13,7 @@
     <reference typeCode="REFR">
       <!-- reference to the relevant measure observation in the eMeasure -->
       <externalObservation classCode="OBS" moodCode="EVN">
-        <id root="<%=population.id %>"/>
+        <id root="<%= id %>"/>
       </externalObservation>
     </reference>
   </observation>

data/templates/cat3/_measure_data.cat3.erb

@@ -56,8 +56,16 @@
         </observation>
       </entryRelationship>
       <% if population.type == 'MSRPOPL' -%>
-      <%== render :partial => 'continuous_variable_value', :locals => {:population => strat} %>
-      <% end -%>
+      <% 
+      #need to lookup the observation population entry and then find the stratification entry for it
+      observ = aggregate_count.populations.find{|p| p.type == "OBSERV"}
+      obs_strat = observ.stratifications.find{|s| s.id == strat.id}
+        if obs_strat
+      %>
+      <%== render :partial => 'continuous_variable_value', :locals => {:id => observ.id, :value=>obs_strat.value } %>
+      <% 
+        end
+      end -%>
       <reference typeCode="REFR">
         <externalObservation classCode="OBS" moodCode="EVN">
           <id root="<%= strat.id %>"/>
@@ -116,7 +124,8 @@
     <% end -%>
   <% end -%>
   <% if population.type == 'MSRPOPL' -%>
-  <%== render :partial => 'continuous_variable_value', :locals => {:population => aggregate_count.populations.find{|p| p.type == "OBSERV"}}%>
+  <%  observ = aggregate_count.populations.find{|p| p.type == "OBSERV"} %>
+  <%== render :partial => 'continuous_variable_value', :locals => {:id => observ.id, :value=>observ.value}%>
   <% end -%>
   <reference typeCode="REFR">
      <externalObservation classCode="OBS" moodCode="EVN">

data/templates/cat3/_organization.cat3.erb

@@ -1,4 +1,3 @@
-<% tag_name ||= "representedOrganization" %>
 <<%=tag_name %>>
   <!-- Represents unique registry organization TIN -->
    <%== render :partial=>"id", :collection=>organization.ids, :id=> "identifier" %>

data/templates/cat3/show.cat3.erb

@@ -41,7 +41,6 @@
   <!-- SHALL -->
   <custodian>
     <assignedCustodian>
-      <%== render :partial=>"id", :collection=>header.custodian.organization.ids, :id=>"identifier" %>
       <%== render :partial=>"organization", :locals=>{organization: header.custodian.organization, tag_name: "representedCustodianOrganization"} %>
     </assignedCustodian>
   </custodian>
@@ -64,7 +63,7 @@
         </name>
      </assignedPerson>
 
-      <%== render :partial=>"organization", :locals=>{organization: header.legal_authenticator.organization} %>
+      <%== render :partial=>"organization", :locals=>{organization: header.legal_authenticator.organization, tag_name: "representedOrganization"} %>
     </assignedEntity>
   </legalAuthenticator>
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: health-data-standards
 version: !ruby/object:Gem::Version
-  version: 3.4.5
+  version: 3.4.6
 platform: ruby
 authors:
 - Andy Gregorowicz
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-04 00:00:00.000000000 Z
+date: 2014-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -286,6 +286,7 @@ files:
 - lib/health-data-standards/models/cqm/bundle.rb
 - lib/health-data-standards/models/cqm/measure.rb
 - lib/health-data-standards/models/cqm/patient_cache.rb
+- lib/health-data-standards/models/cqm/prefilter.rb
 - lib/health-data-standards/models/cqm/query_cache.rb
 - lib/health-data-standards/models/encounter.rb
 - lib/health-data-standards/models/entry.rb