quality-measure-engine 1.1.5 → 2.0.0

data/lib/qme/map/map_reduce_builder.rb

@@ -85,12 +85,15 @@ module QME
         "function (key, value) { 
           var patient = value;
           patient.measure_id = \"#{@measure_def['id']}\";\n"
-        if @params['test_id'] && @params['test_id'].class==BSON::ObjectId
+        if @params['test_id'] && @params['test_id'].class==Moped::BSON::ObjectId
           reduce += "  patient.test_id = new ObjectId(\"#{@params['test_id']}\");\n"
         end
         if @measure_def['sub_id']
           reduce += "  patient.sub_id = \"#{@measure_def['sub_id']}\";\n"
         end
+        if @measure_def['nqf_id']
+          reduce += "  patient.nqf_id = \"#{@measure_def['nqf_id']}\";\n"
+        end
           
         reduce += "patient.effective_date = #{@params['effective_date']};
                    if (patient.provider_performances) {

data/lib/qme/map/map_reduce_executor.rb

@@ -16,7 +16,8 @@ module QME
         @measure_id = measure_id
         @sub_id = sub_id
         @parameter_values = parameter_values
-        determine_connection_information
+        @measure_def = QualityMeasure.new(@measure_id, @sub_id).definition
+        determine_connection_information()
       end
 
       # Examines the patient_cache collection and generates a total of all groups
@@ -24,7 +25,7 @@ module QME
       # collection.
       # @return [Hash] measure groups (like numerator) as keys, counts as values
       def count_records_in_measure_groups
-        patient_cache = get_db.collection('patient_cache')
+        pipeline = []
         base_query = {'value.measure_id' => @measure_id, 'value.sub_id' => @sub_id,
                       'value.effective_date' => @parameter_values['effective_date'],
                       'value.test_id' => @parameter_values['test_id']}
@@ -34,25 +35,37 @@ module QME
         query = base_query.clone
 
         query.merge!({'value.manual_exclusion' => {'$in' => [nil, false]}})
+
+        pipeline << {'$match' => query}
+        pipeline << {'$group' => {
+          "_id" => "$value.measure_id", # we don't really need this, but Mongo requires that we group 
+          "population" => {"$sum" => "$value.population"}, 
+          "denominator" => {"$sum" => "$value.denominator"},
+          "numerator" => {"$sum" => "$value.numerator"},
+          "antinumerator" => {"$sum" => "$value.antinumerator"},
+          "exclusions" => {"$sum" => "$value.exclusions"},
+          "denexcep" => {"$sum" => "$value.denexcep"},
+          "considered" => {"$sum" => 1}
+        }}
         
-        result = {:measure_id => @measure_id, :sub_id => @sub_id, 
+        aggregate = get_db.command(:aggregate => 'patient_cache', :pipeline => pipeline)
+        if aggregate['ok'] != 1
+          raise RuntimeError, "Aggregation Failed"
+        elsif aggregate['result'].size !=1
+          raise RuntimeError, "Expected one group from patient_cache aggregation, got #{aggregate['result'].size}"
+        end
+
+        nqf_id = @measure_def['nqf_id'] || @measure_def['id']
+        result = {:measure_id => @measure_id, :sub_id => @sub_id, :nqf_id => nqf_id, :population_ids => @measure_def["population_ids"],
                   :effective_date => @parameter_values['effective_date'],
                   :test_id => @parameter_values['test_id'], :filters => @parameter_values['filters']}
-        
-         # need to time the old way agains the single query to verify that the single query is more performant
-         aggregate = {"population"=>0, "denominator"=>0, "numerator"=>0, "antinumerator"=>0,  "exclusions"=>0}
-         %w(population denominator numerator antinumerator exclusions).each do |measure_group|
-           patient_cache.find(query.merge("value.#{measure_group}" => true)) do |cursor|
-             aggregate[measure_group] = cursor.count
-           end
-         end
-         aggregate["considered"] = patient_cache.find(query).count
-         aggregate["exclusions"] += patient_cache.find(base_query.merge({'value.manual_exclusion'=>true})).count
-         result.merge!(aggregate)
 
+        result.merge!(aggregate['result'].first)
+        result.reject! {|k, v| k == '_id'} # get rid of the group id the Mongo forced us to use
+        result['exclusions'] += get_db['patient_cache'].find(base_query.merge({'value.manual_exclusion'=>true})).count
         result.merge!(execution_time: (Time.now.to_i - @parameter_values['start_time'].to_i)) if @parameter_values['start_time']
-
-        get_db.collection("query_cache").save(result, safe: true)
+        get_db()["query_cache"].insert(result)
+        get_db().command({:getLastError => 1}) # make sure last insert finished before we continue
         result
       end
 
@@ -60,13 +73,13 @@ module QME
       # in the patient_cache collection. These documents will state the measure groups
       # that the record belongs to, such as numerator, etc.
       def map_records_into_measure_groups
-        qm = QualityMeasure.new(@measure_id, @sub_id)
-        measure = Builder.new(get_db, qm.definition, @parameter_values)
-        records = get_db.collection('records')
-        records.map_reduce(measure.map_function, "function(key, values){return values;}",
-                           :out => {:reduce => 'patient_cache'}, 
-                           :finalize => measure.finalize_function,
-                           :query => {:test_id => @parameter_values['test_id']})
+        measure = Builder.new(get_db(), @measure_def, @parameter_values)
+        get_db().command(:mapReduce => 'records',
+                         :map => measure.map_function,
+                         :reduce => "function(key, values){return values;}",
+                         :out => {:reduce => 'patient_cache'}, 
+                         :finalize => measure.finalize_function,
+                         :query => {:test_id => @parameter_values['test_id']})
         apply_manual_exclusions
       end
       
@@ -74,13 +87,13 @@ module QME
       # This will create a document in the patient_cache collection. This document
       # will state the measure groups that the record belongs to, such as numerator, etc.
       def map_record_into_measure_groups(patient_id)
-        qm = QualityMeasure.new(@measure_id, @sub_id)
-        measure = Builder.new(get_db, qm.definition, @parameter_values)
-        records = get_db.collection('records')
-        records.map_reduce(measure.map_function, "function(key, values){return values;}",
-                           :out => {:reduce => 'patient_cache'}, 
-                           :finalize => measure.finalize_function,
-                           :query => {:medical_record_number => patient_id, :test_id => @parameter_values['test_id']})
+        measure = Builder.new(get_db(), @measure_def, @parameter_values)
+        get_db().command(:mapReduce => 'records',
+                         :map => measure.map_function,
+                         :reduce => "function(key, values){return values;}",
+                         :out => {:reduce => 'patient_cache'}, 
+                         :finalize => measure.finalize_function,
+                         :query => {:medical_record_number => patient_id, :test_id => @parameter_values['test_id']})
         apply_manual_exclusions
       end
       
@@ -88,14 +101,14 @@ module QME
       # This will *not* create a document in the patient_cache collection, instead the
       # result is returned directly.
       def get_patient_result(patient_id)
-        qm = QualityMeasure.new(@measure_id, @sub_id)
-        measure = Builder.new(get_db, qm.definition, @parameter_values)
-        records = get_db.collection('records')
-        result = records.map_reduce(measure.map_function, "function(key, values){return values;}",
-                           :out => {:inline => true}, 
-                           :raw => true, 
-                           :finalize => measure.finalize_function,
-                           :query => {:medical_record_number => patient_id, :test_id => @parameter_values['test_id']})
+        measure = Builder.new(get_db(), @measure_def, @parameter_values)
+        result = get_db().command(:mapReduce => 'records',
+                                  :map => measure.map_function,
+                                  :reduce => "function(key, values){return values;}",
+                                  :out => {:inline => true}, 
+                                  :raw => true, 
+                                  :finalize => measure.finalize_function,
+                                  :query => {:medical_record_number => patient_id, :test_id => @parameter_values['test_id']})
         raise result['err'] if result['ok']!=1
         result['results'][0]['value']
       end
@@ -104,12 +117,11 @@ module QME
       # and sets a flag in each cached patient result for patients that have been excluded from the
       # current measure
       def apply_manual_exclusions
-        exclusions = get_db.collection('manual_exclusions').find({'measure_id'=>@measure_id, 'sub_id'=>@sub_id}).to_a.map do |exclusion|
+        exclusions = get_db()['manual_exclusions'].find({'measure_id'=>@measure_id, 'sub_id'=>@sub_id}).to_a.map do |exclusion|
           exclusion['medical_record_id']
         end
-        get_db.collection('patient_cache').update(
-          {'value.measure_id'=>@measure_id, 'value.sub_id'=>@sub_id, 'value.medical_record_id'=>{'$in'=>exclusions} },
-          {'$set'=>{'value.manual_exclusion'=>true}}, :multi=>true)
+        get_db()['patient_cache'].find({'value.measure_id'=>@measure_id, 'value.sub_id'=>@sub_id, 'value.medical_record_id'=>{'$in'=>exclusions} })
+          .update_all({'$set'=>{'value.manual_exclusion'=>true}})
       end
 
       def filter_parameters
@@ -117,7 +129,7 @@ module QME
         conditions = []
         if(filters = @parameter_values['filters'])
           if (filters['providers'] && filters['providers'].size > 0)
-            providers = filters['providers'].map {|provider_id| BSON::ObjectId(provider_id) if (provider_id and provider_id != 'null') }
+            providers = filters['providers'].map {|provider_id| Moped::BSON::ObjectId(provider_id) if (provider_id and provider_id != 'null') }
             # provider_performances have already been filtered by start and end date in map_reduce_builder as part of the finalize
             conditions << {'value.provider_performances.provider_id' => {'$in' => providers}}
           end

data/lib/qme/map/measure_calculation_job.rb

@@ -1,11 +1,8 @@
 module QME
   module MapReduce
-    # A Resque job that allows for measure calculation by a Resque worker. Can be created as follows:
+    # A delayed_job that allows for measure calculation by a delayed_job worker. Can be created as follows:
     #
-    #    MapReduce::MeasureCalculationJob.create(:measure_id => '0221', :sub_id => 'a', :effective_date => 1291352400, :test_id => xyzzy)
-    #
-    # This will return a uuid which can be used to check in on the status of a job. More details on this can be found
-    # at the {Resque Stats project page}[https://github.com/quirkey/resque-status].
+    #     Delayed::Job.enqueue QME::MapRedude::MeasureCalculationJob.new(:measure_id => '0221', :sub_id => 'a', :effective_date => 1291352400, :test_id => xyzzy)
     #
     # MeasureCalculationJob will check to see if a measure has been calculated before running the calculation. It does
     # this by creating a QME::QualityReport and asking if it has been calculated. If so, it will complete the job without
@@ -13,36 +10,40 @@ module QME
     #
     # When a measure needs calculation, the job will create a QME::MapReduce::Executor and interact with it to calculate
     # the report.
-    class MeasureCalculationJob < Resque::JobWithStatus
-      def perform
-        MeasureCalculationJob.calculate(options)
+    class MeasureCalculationJob
+      attr_accessor :test_id, :measure_id, :sub_id, :effective_date, :filters
+
+      def initialize(options)
+        @measure_id = options['measure_id']
+        @sub_id = options['sub_id']
+        @options = options
       end
       
-      def self.calculate(options)
-        test_id = options['test_id'] ? BSON::ObjectId(options['test_id']) : nil
-        qr = QualityReport.new(options['measure_id'], options['sub_id'], 'effective_date' => options['effective_date'], 'test_id' => test_id, 'filters' => options['filters'])
+      def perform
+        qr = QualityReport.new(@measure_id, @sub_id, @options)
         if qr.calculated?
-          completed("#{options['measure_id']}#{options['sub_id']} has already been calculated") if respond_to? :completed
+          completed("#{@measure_id}#{@sub_id} has already been calculated")
         else
-          map = QME::MapReduce::Executor.new(options['measure_id'], options['sub_id'], 'effective_date' => options['effective_date'], 'test_id' => test_id, 'filters' => options['filters'], 'start_time' => Time.now.to_i)
-
+          map = QME::MapReduce::Executor.new(@measure_id, @sub_id, @options.merge('start_time' => Time.now.to_i))
           if !qr.patients_cached?
-            tick('Starting MapReduce') if respond_to? :tick
+            tick('Starting MapReduce')
             map.map_records_into_measure_groups
-            tick('MapReduce complete') if respond_to? :tick
+            tick('MapReduce complete')
           end
           
-          tick('Calculating group totals') if respond_to? :tick
+          tick('Calculating group totals')
           result = map.count_records_in_measure_groups
-          completed("#{options['measure_id']}#{options['sub_id']}: p#{result['population']}, d#{result['denominator']}, n#{result['numerator']}, e#{result['exclusions']}") if respond_to? :completed
+          completed("#{@measure_id}#{@sub_id}: p#{result['population']}, d#{result['denominator']}, n#{result['numerator']}, excl#{result['exclusions']}, excep#{result['denexcep']}")
         end
+      end
+
+      def completed(message)
+        
+      end
+
+      def tick(message)
         
       end
-      
-#      This can be uncommented and changed to put the jobs on a separate queue.  
-#      def self.queue
-#        :statused
-#      end
     end
   end
 end

data/lib/qme/quality_measure.rb

@@ -8,7 +8,7 @@ module QME
     # @return [Hash] an hash of measure definitions
     def self.all
       result = {}
-      measures = get_db.collection('measures')
+      measures = get_db()['measures']
       measures.find().each do |measure|
         id = measure['id']
         sub_id = measure['sub_id']
@@ -19,7 +19,7 @@ module QME
     end
     
     def self.get_measures(measure_ids)
-      get_db.collection('measures').find('id' => {"$in" => measure_ids})
+      get_db()['measures'].find('id' => {"$in" => measure_ids})
     end
     
     # Creates a new QualityMeasure
@@ -34,11 +34,11 @@ module QME
     # Retrieve a measure definition from the database
     # @return [Hash] a JSON hash of the encoded measure
     def definition
-      measures = get_db.collection('measures')
+      measures = get_db()['measures']
       if @sub_id
-        measures.find_one({'id' => @measure_id, 'sub_id' => @sub_id})
+        measures.find({'id' => @measure_id, 'sub_id' => @sub_id}).first()
       else
-        measures.find_one({'id' => @measure_id})
+        measures.find({'id' => @measure_id}).first()
       end
     end
   end

data/lib/qme/quality_report.rb

@@ -10,25 +10,25 @@ module QME
     # and query_cache collections
     def self.destroy_all
       determine_connection_information
-      get_db.collection("query_cache").drop
-      get_db.collection("patient_cache").drop
+      get_db()["query_cache"].drop
+      get_db()["patient_cache"].drop
     end
     
     # Removes the cached results for the patient with the supplied id and
     # recalculates as necessary
     def self.update_patient_results(id)
-      determine_connection_information
+      determine_connection_information()
       
       # TODO: need to wait for any outstanding calculations to complete and then prevent
       # any new ones from starting until we are done.
 
       # drop any cached measure result calculations for the modified patient
-      get_db.collection("patient_cache").remove('value.medical_record_id' => id)
+      get_db()["patient_cache"].find('value.medical_record_id' => id).remove_all()
       
       # get a list of cached measure results for a single patient
-      sample_patient = get_db.collection('patient_cache').find_one()
+      sample_patient = get_db()['patient_cache'].find().first()
       if sample_patient
-        cached_results = get_db.collection('patient_cache').find({'value.patient_id' => sample_patient['value']['patient_id']})
+        cached_results = get_db()['patient_cache'].find({'value.patient_id' => sample_patient['value']['patient_id']})
         
         # for each cached result (a combination of measure_id, sub_id, effective_date and test_id)
         cached_results.each do |measure|
@@ -42,7 +42,7 @@ module QME
       
       # remove the query totals so they will be recalculated using the new results for
       # the modified patient
-      get_db.collection("query_cache").drop
+      get_db()["query_cache"].drop()
     end
 
     # Creates a new QualityReport
@@ -74,29 +74,47 @@ module QME
     # Kicks off a background job to calculate the measure
     # @return a unique id for the measure calculation job
     def calculate(asynchronous=true)
-      options = {'measure_id' => @measure_id, 'sub_id' => @sub_id, 
-                 'effective_date' => @parameter_values['effective_date'],
-                 'test_id' => @parameter_values['test_id'],
-                 'filters' => QME::QualityReport.normalize_filters(@parameter_values['filters'])}
+      
+      options = {'measure_id' => @measure_id, 'sub_id' => @sub_id}
+      
+      options.merge! @parameter_values
+      options['filters'] = QME::QualityReport.normalize_filters(@parameter_values['filters'])
+      
       if (asynchronous)
-        MapReduce::MeasureCalculationJob.create(options)
+        job = Delayed::Job.enqueue(QME::MapReduce::MeasureCalculationJob.new(options))
+        job._id
       else
-        MapReduce::MeasureCalculationJob.calculate(options)
+        mcj = QME::MapReduce::MeasureCalculationJob.new(options)
+        mcj.perform
       end
     end
     
     # Returns the status of a measure calculation job
     # @param job_id the id of the job to check on
-    # @return [Hash] containing status information on the measure calculation job
+    # @return [Symbol] Will return the status: :complete, :queued, :running, :failed
     def status(job_id)
-      Resque::Status.get(job_id)
+      job = Delayed::Job.where(_id: job_id).first
+      if job.nil?
+        # If we can't find the job, we assume that it is complete
+        :complete
+      else
+        if job.locked_at.nil?
+          :queued
+        else
+          if job.failed?
+            :failed
+          else
+            :running
+          end            
+        end
+      end
     end
     
     # Gets the result of running a quality measure
     # @return [Hash] measure groups (like numerator) as keys, counts as values or nil if
     #                the measure has not yet been calculated
     def result
-      cache = get_db.collection("query_cache")
+      cache = get_db()["query_cache"]
       query = {:measure_id => @measure_id, :sub_id => @sub_id, 
                :effective_date => @parameter_values['effective_date'],
                :test_id => @parameter_values['test_id']}
@@ -106,7 +124,7 @@ module QME
         query.merge!({filters: nil})
       end
         
-      cache.find_one(query)
+      cache.find(query).first()
     end
     
     # make sure all filter id arrays are sorted
@@ -115,11 +133,11 @@ module QME
     end
     
     def patient_result
-      cache = get_db.collection("patient_cache")
+      cache = get_db()["patient_cache"]
       query = {'value.measure_id' => @measure_id, 'value.sub_id' => @sub_id, 
                'value.effective_date' => @parameter_values['effective_date'],
                'value.test_id' => @parameter_values['test_id']}
-      cache.find_one(query)
+      cache.find(query).first()
     end
     
   end

data/lib/qme/railtie.rb

@@ -0,0 +1,7 @@
+module QME
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      Dir[File.join(File.dirname(__FILE__),'tasks/*.rake')].each { |f| load f }
+    end
+  end
+end

data/lib/qme/tasks/bundle.rake

@@ -0,0 +1,14 @@
+require 'quality-measure-engine'
+
+db_name = ENV['DB_NAME'] || 'test'
+
+namespace :bundle do
+  desc 'Import a quality bundle into the database.'
+  task :import, :bundle_path, :delete_existing, :needs => :environment do |task, args|
+    raise "The path to the measures zip file must be specified" unless args.bundle_path
+
+    bundle = File.open(args.bundle_path)    
+    importer = QME::Bundle::Importer.new(db_name)
+    importer.import(bundle, args.delete_existing)
+  end
+end

data/lib/qme/version.rb

@@ -0,0 +1,3 @@
+module QME
+  VERSION = "2.0.0"
+end

data/lib/quality-measure-engine.rb

@@ -1,32 +1,20 @@
 require "bundler/setup"
 
-require 'resque/job_with_status'
+require 'moped'
+require 'delayed_job_mongoid'
+require 'zip/zip'
 
-require_relative 'qme/database_access'
-require_relative 'qme/quality_measure'
+require "qme/version"
+require 'qme/database_access'
+require 'qme/quality_measure'
 
-require_relative 'qme/map/map_reduce_builder'
-require_relative 'qme/map/map_reduce_executor'
-require_relative 'qme/map/measure_calculation_job'
+require 'qme/map/map_reduce_builder'
+require 'qme/map/map_reduce_executor'
+require 'qme/map/measure_calculation_job'
 
-require_relative 'qme/quality_report'
+require 'qme/quality_report'
 
-require_relative 'qme/randomizer/random_patient_creator'
-require_relative 'qme/randomizer/patient_randomizer'
-require_relative 'qme/randomizer/patient_randomization_job'
+require 'qme/bundle/bundle'
+require 'qme/bundle/importer'
 
-require 'singleton'
-require 'health-data-standards'
-
-require_relative 'qme/ext/record'
-
-require_relative 'qme/importer/property_matcher'
-require_relative 'qme/importer/generic_importer'
-require_relative 'qme/importer/measure_properties_generator'
-
-require 'json'
-require 'mongo'
-require 'nokogiri'
-
-require_relative 'qme/measure/measure_loader'
-require_relative 'qme/measure/database_loader'
+require 'qme/railtie' if defined?(Rails)