quality-measure-engine 0.2.0 → 0.8.0

data/lib/qme/map/measure_calculation_job.rb

@@ -0,0 +1,33 @@
+module QME
+  module MapReduce
+    # A Resque job that allows for measure calculation by a Resque worker. Can be created as follows:
+    #
+    #    MapReduce::MeasureCalculationJob.create(:measure_id => '0221', :sub_id => 'a', :effective_date => 1291352400)
+    #
+    # This will return a uuid which can be used to check in on the status of a job. More details on this can be found
+    # at the {Resque Stats project page}[https://github.com/quirkey/resque-status].
+    #
+    # MeasureCalculationJob will check to see if a measure has been calculated before running the calculation. It does
+    # this by creating a QME::QualityReport and asking if it has been calculated. If so, it will complete the job without
+    # running the MapReduce job.
+    #
+    # When a measure needs calculation, the job will create a QME::MapReduce::Executor and interact with it to calculate
+    # the report.
+    class MeasureCalculationJob < Resque::JobWithStatus
+      def perform
+        qr = QualityReport.new(options['measure_id'], options['sub_id'], 'effective_date' => options['effective_date'])
+        if qr.calculated?
+          completed("#{options['measure_id']}#{options['sub_id']} has already been calculated")
+        else
+          map = QME::MapReduce::Executor.new(options['measure_id'], options['sub_id'], 'effective_date' => options['effective_date'])
+          tick('Starting MapReduce')
+          map.map_records_into_measure_groups
+          tick('MapReduce complete')
+          tick('Calculating group totals')
+          result = map.count_records_in_measure_groups
+          completed("#{options['measure_id']}#{options['sub_id']}: p#{result['population']}, d#{result['denominator']}, n#{result['numerator']}, e#{result['exclusions']}")
+        end
+      end
+    end
+  end
+end

data/lib/qme/measure/database_loader.rb

@@ -5,25 +5,11 @@ module QME
   
     # Utility class for working with JSON files and the database
     class Loader
-      # Create a new Loader. Database host and port may be set using the 
-      # environment variables TEST_DB_HOST and TEST_DB_PORT which default
-      # to localhost and 27017 respectively.
+      include DatabaseAccess
+      # Create a new Loader.
       # @param [String] db_name the name of the database to use
-      def initialize(db_name)
-        @db_name = db_name
-        @db_host = ENV['TEST_DB_HOST'] || 'localhost'
-        @db_port = ENV['TEST_DB_PORT'] ? ENV['TEST_DB_PORT'].to_i : 27017
-      end
-      
-      # Lazily creates a connection to the database and initializes the
-      # JavaScript environment
-      # @return [Mongo::Connection]
-      def get_db
-        if @db==nil
-          @db = Mongo::Connection.new(@db_host, @db_port).db(@db_name)
-          QME::MongoHelpers.initialize_javascript_frameworks(@db)
-        end
-        @db
+      def initialize(db_name = nil)
+        determine_connection_information(db_name)
       end
       
       # Load a measure from the filesystem and save it in the database.
@@ -43,8 +29,7 @@ module QME
       # @param [String] collection_name name of the database collection
       # @param [Hash] json the JSON hash to save in the database 
       def save(collection_name, json)
-        collection = get_db.create_collection(collection_name)
-        collection.save(json)
+        get_db[collection_name].save(json)
       end
       
       # Drop a database collection
@@ -53,14 +38,22 @@ module QME
         get_db.drop_collection(collection_name)
       end
       
+      def save_system_js_fn(name, fn)
+        get_db['system.js'].save(
+          {
+            "_id" => name,
+            "value" => BSON::Code.new(fn)
+          }
+        )
+      end
       
       # Save a bundle to the db,  this will save the bundle meta data, javascript extension functions and measures to 
-      # the db in their respective loacations
+      # the db in their respective locations
       # @param [String] bundle_dir the bundle directory
       # @param [String] bundle_collection the collection to save the bundle meta_data and extension functions to
       # @param [String] measure_collection the collection to save the measures to, defaults to measures
-      def save_bundle(bundle_dir,bundle_collection, measure_collection = 'measures')
-        bundle = QME::Measure::Loader.load_bundle(bundle_dir)
+      def save_bundle(bundle_dir, measure_dir, bundle_collection='bundles', measure_collection = 'measures')
+        bundle = QME::Measure::Loader.load_bundle(bundle_dir, measure_dir)
         bundle[:bundle_data][:measures] = []
         b_id = save(bundle_collection,bundle[:bundle_data])
         measures = bundle[:measures]
@@ -70,8 +63,8 @@ module QME
            bundle[:bundle_data][:measures] << m_id
         end
         save(bundle_collection,bundle[:bundle_data])
-        bundle[:bundle_data][:extensions].each do |ext|
-          get_db.eval(ext)
+        bundle[:extensions].each do |name, fn|
+          save_system_js_fn(name, fn)
         end
         bundle
       end

data/lib/qme/measure/measure_loader.rb

@@ -2,6 +2,7 @@ gem 'rubyzip'
 require 'json'
 require 'zip/zip'
 require 'zip/zipfilesystem'
+require File.join(File.dirname(__FILE__),'properties_builder')
 
 module QME
   module Measure
@@ -42,13 +43,22 @@ module QME
         measure_file = File.join(component_dir, collection_def['root'])
         measures = []
         collection_def['combinations'].each do |combination|
+          combination['metadata'] ||= {}
+          if !combination['map_fn']
+            raise "Missing map function in #{collection_file} (sub_id #{combination['metadata']['sub_id']})"
+          end
           map_file = File.join(component_dir, combination['map_fn'])
           measure = load_measure_file(measure_file, map_file)
+          
           # add inline metadata to top level of definition
-          combination['metadata'] ||= {}
           combination['metadata'].each do |key, value|
             measure[key] = value
           end
+          if measure['properties'] && !measure['measure']
+            # load the measure properties if they weren't already added
+            measure['measure'] = get_measure_properties(measure)
+          end
+
           # add inline measure-specific properties to definition
           combination['measure'] ||= {}
           measure['measure'] ||= {}
@@ -65,9 +75,10 @@ module QME
         measures
       end
       
-      # For ease of development, measure definition JSON files and JavaScript 
-      # map functions are stored separately in the file system, this function 
-      # combines the two and returns the result
+      # For ease of development and change mananegment, measure definition JSON
+      # files, property lists and JavaScript map functions are stored separately
+      # in the file system, this function combines the three components and 
+      # returns the result.
       # @param [String] measure_file path to the measure file
       # @param [String] map_fn_file path to the map function file
       # @return [Hash] a JSON hash of the measure with embedded map function.
@@ -75,9 +86,35 @@ module QME
         map_fn = File.read(map_fn_file)
         measure = JSON.parse(File.read(measure_file))
         measure['map_fn'] = map_fn
+        if measure['properties']
+          measure['measure'] = get_measure_properties(measure)
+        end
         measure
       end
       
+      # Load the measure properties from an external file, converting from JSONified XLS if
+      # necessary
+      def self.get_measure_properties(measure)
+        merged_props = {}
+        # normalize field value to be an array of file names
+        if !(measure['properties'].respond_to?(:each))
+          measure['properties'] = [measure['properties']]
+        end
+        measure['properties'].each do |file|
+          measure_props_file = File.join(ENV['MEASURE_PROPS'], file)
+          measure_props = JSON.parse(File.read(measure_props_file))
+          if measure_props['measure']
+            # copy measure properties over
+            merged_props.merge!(measure_props['measure'])
+          else
+            # convert JSONified XLS to properties format and add to measure
+            converted_props = QME::Measure::PropertiesBuilder.build_properties(measure_props, measure_props_file)['measure']
+            merged_props.merge!(converted_props)
+          end
+        end
+        merged_props
+      end
+      
       # Load a JSON file from the specified directory
       # @param [String] dir_path path to the directory containing the JSON file
       # @param [String] filename the JSON file
@@ -89,53 +126,49 @@ module QME
       
       
       #Load a bundle from a directory
-      #@param [String] bundel_path path to directory containing the bundle information
-      
-      def self.load_bundle(bundle_path)
-        bundle = {};
-        bundle_file = File.join(bundle_path,'bundle.js')
-        
-        bundle[:bundle_data] =  File.exists?(bundle_file) ? JSON.parse(File.read(bundle_file)) : JSON.parse("{}")
-        bundle[:bundle_data][:extensions] = load_bundle_extensions(bundle_path)
-        bundle[:measures] = []
-        Dir.glob(File.join(bundle_path, 'measures', '*')).each do |measure_dir|
-          load_measure(measure_dir).each do |measure|
-            bundle[:measures] << measure
+      # @param [String] bundle_path path to directory containing the bundle information
+      def self.load_bundle(bundle_path, measure_dir)
+        begin
+          bundle = {};
+          bundle_file = File.join(bundle_path,'bundle.json')
+          license_file = File.join(bundle_path, 'license.html')
+          
+          bundle[:bundle_data] =  File.exists?(bundle_file) ? JSON.parse(File.read(bundle_file)) : JSON.parse("{}")
+          bundle[:bundle_data][:license] = File.exists?(license_file) ? File.read(license_file) : ""
+          bundle[:extensions] = load_bundle_extensions(bundle_path)
+          bundle[:bundle_data][:extensions] = bundle[:extensions].keys
+          bundle[:measures] = []
+          Dir.glob(File.join(bundle_path, measure_dir, '*')).each do |measure_dir|
+            load_measure(measure_dir).each do |measure|
+              bundle[:measures] << measure
+            end
           end
+          bundle
+        rescue Exception => e
+          print e.backtrace.join("\n")
+          throw e
         end
-        bundle
       end
       
       
       # Load all of the extenson functions that will be available to map reduce functions from the bundle dir
       # This will load from bundle_path/js and from ext directories in the individule measures directories 
       # like bundle_path/measures/0001/ext/ 
-      #@param [String] bundle_path the path to the bundle directory
+      # @param [String] bundle_path the path to the bundle directory
+      # @return [Hash] name, function
       def self.load_bundle_extensions(bundle_path)
-        extensions = []
-        Dir.glob(File.join(bundle_path, 'js', '*.js')).each do |js_file|
+        extensions = {}
+        process_extension = lambda do |js_file|
+          fn_name = File.basename(js_file, ".js")
           raw_js = File.read(js_file)
-           extensions << raw_js
-        end
-        Dir.glob(File.join(bundle_path, 'measures', '*','ext', '*.js')).each do |js_file|
-          raw_js = File.read(js_file)
-          extensions << raw_js
+          extensions[fn_name] = raw_js
         end
+        Dir.glob(File.join(File.dirname(__FILE__), '../../..', 'js', '*.js')).each &process_extension
+        Dir.glob(File.join(bundle_path, 'js', '*.js')).each &process_extension
+        Dir.glob(File.join(bundle_path, 'measures', '*','ext', '*.js')).each &process_extension
         extensions
       end
       
-      
-      def self.load_from_zip(archive)
-            unzip_path = "./tmp/#{Time.new.to_i}/" 
-            FileUtils.mkdir_p(unzip_path)
-            all_measures = []
-            Zip::ZipFile.foreach(archive) do |zipfile|
-              fname = unzip_path+ zipfile.name
-              FileUtils.rm fname, :force=>true
-              zipfile.extract(fname)
-            end
-           load_bundle(unzip_path)
-      end
     end
   end
 end

data/lib/qme/measure/properties_builder.rb

@@ -0,0 +1,184 @@
+module QME
+  module Measure
+  
+    # Utility class for converting JSONified XLS file to measure properties
+    class PropertiesBuilder
+    
+      GROUPING = 'GROUPING'
+      STANDARD_TAXONOMY = 'standard_taxonomy'
+      STANDARD_CODE_LIST = 'standard_code_list'
+      STANDARD_CONCEPT = 'standard_concept'
+      STANDARD_CONCEPT_ID = 'standard_concept_id'
+      STANDARD_CATEGORY = 'standard_category'
+      QDS_DATA_TYPE = 'QDS_data_type'
+      PROPERTIES_TO_IGNORE = %w(birthdate measurement_enddate measurement_period)
+      
+      # Convert JSONified measure XLS to measure properties format
+      def self.build_properties(xls_json, xls_file)
+        xls_json = patch_properties(xls_json, xls_file)
+        
+        # build groupings
+        grouped_json = build_groups(xls_json)
+        
+        properties = {}
+        grouped_json.each do |value|
+          property_value = build_property(value)
+          if !PROPERTIES_TO_IGNORE.include?(property_value[STANDARD_CONCEPT])
+            properties[property_name(value)] = property_value
+          end
+        end
+        
+        {'measure' => properties}
+      end
+      
+      # Build the top-level JSON hash for a measure property
+      def self.build_property(parent)
+        property = {}
+        property[STANDARD_CONCEPT] = build_underscore_separated_name(parent[STANDARD_CONCEPT])
+        property[STANDARD_CATEGORY] = build_underscore_separated_name(parent[STANDARD_CATEGORY])
+        property[STANDARD_CONCEPT_ID] = parent[STANDARD_CONCEPT_ID]
+        property['qds_data_type'] = build_underscore_separated_name(parent[QDS_DATA_TYPE])
+        property['qds_id'] = parent['QDS_id']
+        property['type'] = 'array'
+        property['items'] = build_item_def(parent)
+        property['codes'] = build_code_list(parent)
+        property
+      end
+      
+      # Build the definition of a property item
+      def self.build_item_def(property)
+        if property['value']
+          {
+            'type' => 'object',
+            'properties' => {
+              'value' => {
+                'description' => property['value']['description'],
+                'type' => property['value']['type']
+              },
+              'date' => {
+                'description' => property['value']['date_description'],
+                'type' => 'number',
+                'format' => 'utc-sec'
+              }
+            }
+          }
+        elsif property['range']
+          {
+            'type' => 'object',
+            'properties' => {
+              'start' => {
+                'description' => property['range']['start_description'],
+                'type' => 'number',
+                'format' => 'utc-sec'
+              },
+              'end' => {
+                'description' => property['range']['end_description'],
+                'type' => 'number',
+                'format' => 'utc-sec'
+              }
+            }
+          }
+        else
+          {
+            'type' => 'number',
+            'format' => 'utc-sec'
+          }
+        end
+      end
+      
+      # Build the set of code lists for a property
+      def self.build_code_list(property)
+        if property['child_nodes']
+          property['child_nodes'].collect { |item| build_code_def(item) }
+        else
+          [build_code_def(property)]
+        end
+      end
+      
+      # Build a single code list JSON hash for a property
+      def self.build_code_def(property)
+        {
+          'set' => property['standard_taxonomy'],
+          'version' => property['standard_taxonomy_version'],
+          STANDARD_CONCEPT => build_underscore_separated_name(property[STANDARD_CONCEPT]),
+          STANDARD_CONCEPT_ID => property[STANDARD_CONCEPT_ID],
+          'qds_id' => property['QDS_id'],
+          'values' => extract_code_values(property['standard_code_list'], property['standard_taxonomy'])
+        }
+      end
+      
+      # Extract the code values from their string form into an array of strings, one per code
+      def self.extract_code_values(string_list, set)
+        return [] if string_list==nil || string_list.length==0
+        string_list.split(',').collect do |entry|
+          if set=='CPT' && entry.include?('-')
+            # special handling for ranges in CPT code sets, e.g. 10010-10015
+            eval(entry.strip.gsub('-','..')).to_a.collect { |i| i.to_s }
+          else
+            entry.strip
+          end
+        end.flatten
+      end
+      
+      # Generate a property name from the standard concept and qds data type
+      def self.property_name(property)
+        build_underscore_separated_name(property[STANDARD_CONCEPT], property[QDS_DATA_TYPE])
+      end
+      
+      # Break all the supplied strings into separate words and return the resulting list as a
+      # new string with each word separated with '_'
+      def self.build_underscore_separated_name(*components)
+        name = []
+        components.each do |component|
+          name.concat component.gsub(/\W/,' ').split.collect { |word| word.strip.downcase }
+        end
+        name.join '_'
+      end
+      
+      # build an array of grouped properties
+      def self.build_groups(xls_json)
+        grouped_json = []
+        exclude_child_ids = []
+        
+        # add grouped members
+        groups = xls_json.values.select { |value| value[STANDARD_TAXONOMY]!=nil && value[STANDARD_TAXONOMY].upcase==GROUPING }
+        groups.each do |group|
+          group_child_ids = group[STANDARD_CODE_LIST].split(',').collect { |id| id.strip }
+          children = xls_json.values.select do |child| 
+            group_child_ids.include?(child[STANDARD_CONCEPT_ID].strip)
+          end
+          group['child_nodes'] = children
+          grouped_json << group
+          if group['group_type']==nil || group['group_type']!='abstract'
+            exclude_child_ids.concat group_child_ids
+          end
+        end
+
+        # add remaining non-grouped members
+        xls_json.values.each do |value|
+          if value[STANDARD_TAXONOMY]!=GROUPING && !exclude_child_ids.include?(value[STANDARD_CONCEPT_ID].strip)
+            grouped_json<<value
+          end
+        end
+        grouped_json
+      end
+      
+      # Patch the supplied property_json hash with the contents of a patch file named
+      # "#{property_file}.patch" and return the result
+      def self.patch_properties(property_json, property_file)
+        patch_file = "#{property_file}.patch"
+        
+        if File.exists? patch_file
+          patch_json = JSON.parse(File.read(patch_file))
+        
+          property_json = property_json.merge(patch_json) do |key, old, new|
+            old.merge(new)
+          end
+        end
+        property_json
+      end
+      
+    end
+      
+  end
+end