quality-measure-engine 0.2.0 → 0.8.0

data/lib/qme/measure/properties_converter.rb

@@ -0,0 +1,27 @@
+require 'roo'
+
+module QME
+  module Measure
+  
+    # Utility class for converting NQF XLS files to JSON
+    class Converter
+    
+      #Convert an NQF XLS file to a hash
+      def self.from_xls(file)
+        xls =Excelx.new(file)
+        xls.default_sheet='Measure_QDS'
+        result = {}
+        (xls.header_line+1..xls.last_row).each do |row|
+          entry = {}
+          (xls.first_column..xls.last_column).each do |column|
+            entry[xls.cell(xls.header_line, column)] = xls.cell(row, column)
+          end
+          result[entry['QDS_id']] = entry
+        end
+        result
+      end
+      
+    end
+    
+  end
+end

data/lib/qme/quality_measure.rb

@@ -0,0 +1,41 @@
+module QME
+  class QualityMeasure
+    include DatabaseAccess
+    extend DatabaseAccess
+    determine_connection_information
+    
+    # Return a list of the measures in the database
+    # @return [Hash] an hash of measure definitions
+    def self.all
+      result = {}
+      measures = get_db.collection('measures')
+      measures.find().each do |measure|
+        id = measure['id']
+        sub_id = measure['sub_id']
+        measure_id = "#{id}#{sub_id}.json"
+        result[measure_id] ||= measure
+      end
+      result
+    end
+    
+    # Creates a new QualityMeasure
+    # @param [String] measure_id value of the measure's id field
+    # @param [String] sub_id value of the measure's sub_id field, may be nil for measures with only a single numerator and denominator
+    def initialize(measure_id, sub_id = nil)
+      @measure_id = measure_id
+      @sub_id = sub_id
+      determine_connection_information
+    end
+    
+    # Retrieve a measure definition from the database
+    # @return [Hash] a JSON hash of the encoded measure
+    def definition
+      measures = get_db.collection('measures')
+      if @sub_id
+        measures.find_one({'id' => @measure_id, 'sub_id' => @sub_id})
+      else
+        measures.find_one({'id' => @measure_id})
+      end
+    end
+  end
+end

data/lib/qme/quality_report.rb

@@ -0,0 +1,61 @@
+module QME
+  # A class that allows you to create and obtain the results of running a
+  # quality measure against a set of patient records.
+  class QualityReport
+    include DatabaseAccess
+    extend DatabaseAccess
+    determine_connection_information
+
+    # Gets rid of all calculated QualityReports by dropping the patient_cache
+    # and query_cache collections
+    def self.destroy_all
+      determine_connection_information
+      get_db.collection("query_cache").drop
+      get_db.collection("patient_cache").drop
+    end
+
+    # Creates a new QualityReport
+    # @param [String] measure_id value of the measure's id field
+    # @param [String] sub_id value of the measure's sub_id field, may be nil 
+    #                 for measures with only a single numerator and denominator
+    # @param [Hash] parameter_values slots in the measure definition that need to
+    #               be filled in.
+    def initialize(measure_id, sub_id, parameter_values)
+      @measure_id = measure_id
+      @sub_id = sub_id
+      @parameter_values = parameter_values
+      determine_connection_information
+    end
+    
+    # Determines whether the quality report has been calculated for the given
+    # measure and parameters
+    # @return [true|false]
+    def calculated?
+      ! result().nil?
+    end
+    
+    # Kicks off a background job to calculate the measure
+    # @return a unique id for the measure calculation job
+    def calculate
+      MapReduce::MeasureCalculationJob.create(:measure_id => @measure_id, :sub_id => @sub_id, 
+                                              :effective_date => @parameter_values['effective_date'])
+    end
+    
+    # Returns the status of a measure calculation job
+    # @param job_id the id of the job to check on
+    # @return [Hash] containing status information on the measure calculation job
+    def status(job_id)
+      Resque::Status.get(job_id)
+    end
+    
+    # Gets the result of running a quality measure
+    # @return [Hash] measure groups (like numerator) as keys, counts as values or nil if
+    #                the measure has not yet been calculated
+    def result
+      cache = get_db.collection("query_cache")
+      query = {:measure_id => @measure_id, :sub_id => @sub_id, 
+               :effective_date => @parameter_values['effective_date']}
+      cache.find_one(query)
+    end
+  end
+end

data/lib/quality-measure-engine.rb

@@ -1,29 +1,32 @@
 Bundler.require(:default)
 
-LIB = File.dirname(__FILE__)
+require 'resque/job_with_status'
 
-require LIB + '/qme/map/map_reduce_builder'
-require LIB + '/qme/map/map_reduce_executor'
+require_relative 'qme/database_access'
+require_relative 'qme/quality_measure'
 
-require LIB + '/qme/randomizer/patient_randomizer'
+require_relative 'qme/map/map_reduce_builder'
+require_relative 'qme/map/map_reduce_executor'
+require_relative 'qme/map/measure_calculation_job'
 
-require 'singleton'
+require_relative 'qme/quality_report'
+
+require_relative 'qme/randomizer/patient_randomizer'
 
-require LIB + '/qme/importer/entry'
-require LIB + '/qme/importer/property_matcher'
-require LIB + '/qme/importer/patient_importer'
-require LIB + '/qme/importer/code_system_helper'
-require LIB + '/qme/importer/hl7_helper'
+require 'singleton'
 
-require LIB + '/qme/importer/section_importer'
-require LIB + '/qme/importer/generic_importer'
+require_relative 'qme/importer/entry'
+require_relative 'qme/importer/property_matcher'
+require_relative 'qme/importer/patient_importer'
+require_relative 'qme/importer/code_system_helper'
+require_relative 'qme/importer/hl7_helper'
 
-require LIB + '/qme/mongo_helpers'
+require_relative 'qme/importer/section_importer'
+require_relative 'qme/importer/generic_importer'
 
 require 'json'
 require 'mongo'
 require 'nokogiri'
 
-require LIB + '/qme/measure/measure_loader'
-require LIB + '/qme/measure/database_loader'
-
+require_relative 'qme/measure/measure_loader'
+require_relative 'qme/measure/database_loader'

data/lib/tasks/fixtures.rake

@@ -0,0 +1,91 @@
+require 'json'
+require 'erb'
+
+fixtures_dir = ENV['FIXTURE_DIR'] || File.join('fixtures', 'measures')
+output_dir = ENV['HTML_OUTPUT_DIR'] || File.join('tmp', 'html')
+
+def epoch_pp(seconds_since_epoch)
+  if seconds_since_epoch.kind_of? Fixnum
+    Time.at(seconds_since_epoch).utc.strftime('%B %d, %Y')
+  else
+    seconds_since_epoch
+  end
+end
+
+def extract_dates(thingy, dates)
+  if thingy.kind_of? Fixnum
+    dates[thingy] = epoch_pp(thingy)
+  elsif thingy.kind_of? Array
+    thingy.each {|inner_thingy| extract_dates(inner_thingy, dates)}
+  elsif thingy.kind_of? Hash
+    thingy.values.each {|inner_thingy| extract_dates(inner_thingy, dates)}
+  else
+    # Must be a string
+  end
+end
+
+namespace :fixtures do
+  
+  # NOTE: This will create invalid JSON fixture files, since JSON does not have the ability add comments.
+  #       However, most JSON parsers handle this just fine.
+  desc 'Place a comment header at the top of each fixture file with a table of seconds since epoch to month, day, year'
+  task :time_comments do
+    Dir.glob(File.join(fixtures_dir,'*','patients','*.json')) do |file|
+      fixture_person = JSON.parse(File.read(file))
+      dates = {}
+      extract_dates(fixture_person, dates)
+      
+      fixture = File.open(file, 'w')
+      fixture.write("//\n// Dates used in this fixture\n//\n")
+      dates.each_pair do |key, value|
+        fixture.write("// #{key} - #{value}\n")
+      end
+      fixture.write(JSON.pretty_generate(fixture_person))
+      fixture.close
+    end
+  end
+  
+  desc 'Generate HTML versions of the patient fixtures'
+  task :html_doc do
+    patient_template = File.read(File.join(File.dirname(__FILE__), 'patient_fixture.erb'))
+    patient_erb = ERB.new(patient_template)
+    
+    result_template = File.read(File.join(File.dirname(__FILE__), 'result.erb'))
+    result_erb = ERB.new(result_template)
+    
+    fixtures_path = Pathname.new(fixtures_dir)
+    output_path = Pathname.new(output_dir)
+    
+    Dir.glob(File.join(fixtures_dir,'*','patients','*.json')) do |file|
+      @patient = JSON.parse(File.read(file))
+      @filename = file
+      html = patient_erb.result(binding)
+      
+      fixture_path = Pathname.new(file)
+      relative_fixture_path = fixture_path.relative_path_from(fixtures_path)
+      measure_output_path = output_path + relative_fixture_path
+      FileUtils.mkdir_p(measure_output_path.dirname)
+      File.open(measure_output_path.sub_ext('.html'), 'w') do |html_out|
+        html_out.write(html)
+      end
+    end
+    
+    Dir.glob(File.join(fixtures_dir,'*','result.json')) do |file|
+      result_json = JSON.parse(File.read(file))
+      if result_json["results"]
+        @results = result_json["results"]
+      else
+        @results = [result_json]
+      end
+      html = result_erb.result(binding)
+      
+      result_path = Pathname.new(file)
+      relative_result_path = result_path.relative_path_from(fixtures_path)
+      result_output_path = output_path + relative_result_path
+      
+      File.open(result_output_path.sub_ext('.html'), 'w') do |html_out|
+        html_out.write(html)
+      end
+    end
+  end
+end

data/lib/tasks/measure.rake

@@ -5,19 +5,18 @@ require 'zlib'
 gem 'rubyzip'
 require 'zip/zip'
 require 'zip/zipfilesystem'
-
-
 require File.join(path,'../quality-measure-engine')
 
-
 measures_dir = ENV['MEASURE_DIR'] || 'measures'
-puts "Loading measures from #{measures_dir}"
-
 bundle_dir = ENV['BUNDLE_DIR'] || './'
+xls_dir = ENV['XLS_DIR'] || 'xls'
+db_name = ENV['DB_NAME'] || 'test'
+
 namespace :measures do
   
   desc 'Build all measures to tmp directory'
   task :build do
+    puts "Loading measures from #{measures_dir}"
     dest_dir = File.join('.', 'tmp')
     Dir.mkdir(dest_dir) if !Dir.exist?(dest_dir)
     Dir.glob(File.join(measures_dir, '*')).each do |measure_dir|
@@ -34,43 +33,63 @@ namespace :measures do
     end
   end
   
-  
-  
   desc "run the map_test tool"
   task :map_tool do
+    puts "Loading measures from #{measures_dir}"
     require File.join(path,"../../map_test/map_test.rb")
   end
   
-  
-  desc "bundle measures into a compressed file for deployment"
-  task :bundle do
+  desc 'Take a snapshot of the current measures, system.js and bundles collections and store as a ZIP file'
+  task :snapshot do
+    tmp = File.join('.', 'tmp')
+    dest_dir = File.join(tmp, 'bundle')
+    FileUtils.rm_r dest_dir, :force=>true
+    FileUtils.mkdir_p(dest_dir)
+    system("mongodump --db #{db_name} --collection bundles --out - > #{dest_dir}/bundles.bson")
+    system("mongodump --db #{db_name} --collection system.js --out - > #{dest_dir}/system.js.bson")
+    system("mongodump --db #{db_name} --collection measures --out - > #{dest_dir}/measures.bson")
+    read_me = <<EOF
+Load the included files into Mongo as follows:
+
+mongorestore --db dbname --drop measures.bson
+mongorestore --db dbname --drop bundles.bson
+mongorestore --db dbname --drop system.js.bson
+
+Where dbname is the name of the database you want to load the measures into. For a
+production system this will typically be pophealth-production. For a development
+system it will typically be pophealth-development.
+
+Note that the existing contents of the destination database's measures, system.js and
+bundles collections will be lost.
+EOF
+    File.open(File.join(dest_dir, 'README.txt'), 'w') {|f| f.write(read_me) }
     
-      md = File.join(bundle_dir,measures_dir)
-      js = File.join(bundle_dir,'js')
-      bf = File.join(bundle_dir,'bundle.js')
-      
-      tmp = './tmp'
-      bundle_tmp = File.join(tmp,'bundle')
-      
-      md.sub!(%r[/$],'')
-      FileUtils.rm bundle_tmp, :force=>true
-      FileUtils.mkdir_p(bundle_tmp)
-      archive = File.join(tmp,'bundle.zip')
-      FileUtils.rm archive, :force=>true
-     
-       Zip::ZipFile.open(archive, 'w') do |zipfile|
-          Dir["#{md}/**/**"].reject{|f|f==archive}.each do |file|
-           zipfile.add(file.sub(bundle_dir,''),file)
-          end
-                
-          Dir["#{js}/**/**"].reject{|f|f==archive}.each do |file|
-             zipfile.add(file.sub(bundle_dir,''),file)
-          end
-           
-         if File.exists?(bf)
-           zipfile.add(bf.sub(bundle_dir,''),bf)
-         end
-       end
+    archive = File.join(tmp, 'bundle.zip')
+    puts "Snapshot saved to #{archive}"
+    FileUtils.rm archive, :force=>true
+    
+    Zip::ZipFile.open(archive, 'w') do |zipfile|
+      Dir["#{dest_dir}/*"].each do |file|
+        zipfile.add(File.basename(file),file)
+      end
+    end    
+    FileUtils.rm_r dest_dir, :force=>true
+  end
+  
+  desc "convert NQF Excel spreadsheets to JSON"
+  task :convert do
+    require LIB + '/qme/measure/properties_builder'
+    require LIB + '/qme/measure/properties_converter'
+    dest_dir = File.join('.', 'tmp')    
+    Dir.mkdir(dest_dir) if !Dir.exist?(dest_dir)
+    Dir.glob(File.join(xls_dir, '*.xlsx')).each do |measure|
+      properties = QME::Measure::Converter.from_xls(measure)
+      json = JSON.pretty_generate(properties)
+      file_name = File.join(dest_dir, "#{File.basename(measure)}.json")
+      file = File.new(file_name,  "w")
+      file.write(json)
+      file.close
+    end
   end
 
 end

data/lib/tasks/mongo.rake

@@ -2,19 +2,21 @@ path = File.dirname(__FILE__)
 path = path.index('lib') == 0 ? "./#{path}" : path
 require 'mongo'
 require 'json'
+require 'resque'
 require File.join(path,'../quality-measure-engine')
 
 measures_dir = ENV['MEASURE_DIR'] || 'measures'
 bundle_dir = ENV['BUNDLE_DIR'] || '.'
 fixtures_dir = ENV['FIXTURE_DIR'] || File.join('fixtures', 'measures')
 db_name = ENV['DB_NAME'] || 'test'
-loader = QME::Database::Loader.new(db_name)
+loader = QME::Database::Loader.new()
 
 namespace :mongo do
 
   desc 'Removed cached measure results'
   task :drop_cache do
     loader.drop_collection('query_cache')
+    loader.drop_collection('patient_cache')
   end
   
   desc 'Remove the patient records collection'
@@ -23,22 +25,11 @@ namespace :mongo do
   end
 
   desc 'Remove the measures and bundles collection'
-  task :drop_bundle => :drop_measures  do
+  task :drop_bundle do
     loader.drop_collection('bundles')
-  end
-  
-  desc 'Remove the measures collection'
-  task :drop_measures => :drop_cache do
     loader.drop_collection('measures')
   end
   
-  desc 'Remove all measures and reload'
-  task :reload_measures => :drop_measures do
-    Dir.glob(File.join(measures_dir, '*')).each do |measure_dir|
-      loader.save_measure(measure_dir, 'measures')
-    end
-  end
-  
   desc 'Remove all patient records and reload'
   task :reload_records => :drop_records do
     load_files(loader, File.join(fixtures_dir,'*','patients','*.json'), 'records')
@@ -46,21 +37,24 @@ namespace :mongo do
 
   desc 'Remove all patient records and reload'
   task :reload_bundle => [:drop_bundle] do
-   loader.save_bundle(bundle_dir,'bundles')
+   loader.save_bundle(bundle_dir, measures_dir)
   end
   
   desc 'Clear database and road each measure and its sample patient files'
-  task :reload => [:reload_records, :reload_measures]
+  task :reload => [:reload_records, :reload_bundle]
   
   desc 'Seed the query cache by calculating the results for all measures'
   task :seed_cache, [:year, :month, :day] do |t, args|
+    db = loader.get_db
+    patient_cache = db['patient_cache']
+    patient_cache.create_index([['value.measure_id', Mongo::ASCENDING],
+                                ['value.sub_id', Mongo::ASCENDING],
+                                ['value.effective_date', Mongo::ASCENDING]])
     year = args.year.to_i>0 ? args.year.to_i : 2010
     month = args.month.to_i>0 ? args.month.to_i : 9
     day = args.day.to_i>0 ? args.day.to_i : 19
-    map = QME::MapReduce::Executor.new(loader.get_db)
-    map.all_measures.each_value do |measure_def|
-      result = map.measure_result(measure_def['id'], measure_def['sub_id'], :effective_date=>Time.gm(year, month, day).to_i)
-      puts "#{measure_def['id']}#{measure_def['sub_id']}: p#{result[:population]}, d#{result[:denominator]}, n#{result[:numerator]}, e#{result[:exclusions]}"
+    QME::QualityMeasure.all.each_value do |measure_def|
+      QME::MapReduce::MeasureCalculationJob.create(:measure_id => measure_def['id'], :sub_id => measure_def['sub_id'], :effective_date => Time.gm(year, month, day).to_i)
     end
   end