pvdgm_services 0.1.0

data/app/models/services/mds_assessment.rb

@@ -0,0 +1,371 @@
+module Services
+
+  class MdsAssessment < ActiveRecord::Base
+    include MdsAssessmentCategorizer
+
+    belongs_to :mds_upload
+    belongs_to :facility
+    belongs_to :resident
+    belongs_to :stay
+
+    attr_encrypted :responses, type: 'json'
+
+    VERSION1_0 = "1.00"
+    VERSION1_1 = "1.10"
+    VERSION1_1_DATE = "20120401"
+    CORRECTED = 1
+
+    MATCH_MAP = { 
+      'x0150' => 'a0200',
+      'x0600a' => 'a0310a',
+      'x0600b' => 'a0310b',
+      'x0600c' => 'a0310c',
+      'x0600d' => 'a0310d',
+      'x0600f' => 'a0310f',
+      'x0700a' => 'a2300',
+      'x0700b' => 'a2000',
+      'x0700c' => 'a1600'
+    }
+
+    MISSING_ATTRIBUTE_VALUES = %w{ ^ - }
+    CLEAN_ATTR_METHOD_REX = /^clean_(\w+)$/
+
+    # Record types
+    ADD_NEW_RECORD = 1
+    MODIFY_EXISTING_RECORD = 2
+    INACTIVATE_EXISTING_RECORD = 3
+
+    # These are fields that must be treated as if they are
+    # full-fledged attributes of the assessment. If they are
+    # not listed here, any access to the field will raise an
+    # exception from #method_missing.
+    VALIDATED_FIELDS = [ :itm_sbst_cd,
+                         :fac_id,
+                         :corrected,
+                         :prior_isc_cd,
+                         :state_cd,
+                         :a0050,
+                         :a0100b,
+                         :a0200,
+                         :a1600,
+                         :a1800,
+                         :a2000,
+                         :a2200,
+                         :a2300,
+                         :a0310a,
+                         :a0310b,
+                         :a0310c,
+                         :a0310d,
+                         :a0310f,
+                         :a0900,
+                         :k0200a,
+                         :k0200b,
+                         :x0100]
+
+    CREATED = 0
+    PROCESSED = 1
+
+    alias_attribute :assessment_date, :reference_date
+
+    validates :responses, 
+      presence: true
+
+    validates :fac_id, 
+      presence: { 
+        :message => "FAC_ID (CMS Assigned facility submission id) cannot be blank.",
+        if: :has_responses? }
+
+    validates :itm_sbst_cd, 
+      presence: { 
+        message: "ITM_SBST_CD (ISC) cannot be blank.", 
+        if: :has_responses? },
+      isc_code: {
+        if: :has_responses? }
+
+    validates :state_cd, 
+      presence: { 
+        message: "Invalid state code entered in state_cd.",
+        if: :has_responses? },
+      state_code: {
+        if: :has_responses? }
+
+    validates :a1600, 
+      mds_date: { 
+        unless: :inactivation_assessment?,
+        if: :has_responses? }
+
+    validates :a2000, 
+      mds_date: { if: :discharge_or_death_assessment? }
+
+    validates :a2300, 
+      mds_date: { if: :a0310f_99? }
+
+    validates :a0900,  
+      mds_birthdate: {
+        msg_invalid_characters: "Invalid birth date entered in a0900. Birth date may contain only numbers.",
+        msg_invalid_format: "Invalid birth date entered in a0900. Birth date must be formatted as YYYYMMDD, YYYYMM or YYYY.",
+        msg_invalid: "Invalid birth date entered in a0900",
+        if: :has_responses? }
+
+    validates :k0200a, 
+      mds_integer: { 
+        minimum: 0,
+        maximum: 99,
+        message: "Invalid height entered in k0200a.  Height must be between 0 and 99.",
+        if: :has_responses? }
+
+    validates :k0200b, 
+      mds_integer: {
+        minimum: 0,
+        maximum: 999,
+        message: "Invalid weight entered in k0200b.  Weight must be between 0 and 999.",
+        if: :has_responses? }
+
+    validates :version, 
+      mds_version: {
+        if: :has_responses? }
+
+    validate :twisted_discharge_dates, 
+        if: :discharge_or_death_assessment?
+
+    before_validation :determine_reference_date, unless: :has_reference_date?
+    before_validation :determine_version, if: :has_reference_date?
+    before_validation :determine_matching_fields, if: :has_reference_date?
+    before_create :set_inact_corr_fields
+
+    scope :noncorrected, -> { where("corrected is null OR corrected = 0") }
+
+    scope :correctable_assessments, ->(mds_id, resident_id) { 
+      where(resident_id: resident_id).
+      where("id != ?", mds_id).
+      where("corrected != ? or corrected IS NULL", CORRECTED) }
+
+    scope :correction_assessments_for_target, ->(mds) {
+      where(resident_id: mds.resident_id).
+        where(type_of_record: 2).
+        where("
+              (previous_reference_date = '#{mds.a2300}' AND item_subset_code = '#{mds.item_subset_code}') OR
+              (inact_corr_match = '#{mds.inact_corr_match}')
+              ".compact).
+        order("reference_date DESC")
+    }
+
+    scope :inactivation_assessments_for_target, ->(mds) {
+      where(resident_id: mds.resident_id).
+        where(item_subset_code: IscCode::INACTIVATION_CODE).
+        where("
+              (previous_reference_date = '#{mds.a2300}' AND item_subset_code = '#{mds.item_subset_code}') OR
+              (inact_corr_match = '#{mds.inact_corr_match}')
+              ".compact).
+        order("reference_date DESC")
+    }
+
+    # Paired with method_missing
+    def respond_to_missing?(sym, include_private=false)
+      VALIDATED_FIELDS.include?(sym) || (responses.present? && responses.keys.include?(sym.to_s)) || super
+    end
+
+    def method_missing(sym, *args)
+      # If an attribute is referenced as mds.clean_* (e.g. mds.clean_k0200b), the value 
+      # we will return the value of the attribute as nil if it is one of the special
+      # mds values (^|-). If we do it this way, we don't have to splatter the special
+      # value junk all over the code base.
+      if sym.to_s =~ CLEAN_ATTR_METHOD_REX
+        return filter_value(responses.present? ? responses[$1] : nil)
+      end
+      return responses.present? ? responses[sym.to_s] : nil if VALIDATED_FIELDS.include?(sym)
+      return responses[sym.to_s] if responses.present? && responses.keys.include?(sym.to_s)
+      super
+    end
+
+    def unit_id
+      MDS_MODULE_ID
+    end
+
+    def has_responses?
+      responses.present?
+    end
+
+    def version1_0?
+      reference_date <  Date.parse("20120401")
+    end
+
+    def version1_1?
+      reference_date >= Date.parse("20120401")
+    end
+
+    def entry_type
+      (clean_a1700 == "2") ? Stay::REENTRY : Stay::ADMISSION
+    end
+
+    def entry_date 
+      a1600
+    end
+
+    def discharge_date
+      a2000
+    end
+
+    def discharge_type
+      clean_a2100.blank? ? nil : clean_a2100.to_i
+    end
+
+    def ssn 
+      clean_a0600a || clean_x0500 || ""
+    end
+
+    def first_name
+     clean_a0500a || clean_x0200a || "" 
+    end
+
+    def last_name
+      clean_a0500c || clean_x0200c || ""
+    end
+
+    def gender
+      clean_a0800 || clean_x0300 || ""
+    end
+
+    def comatose?
+      clean_b0100 == "1"
+    end
+
+    def inact_corr_match_fields(entry_discharge_reporting)
+      fields = %w{ x0150 x0600a x0600b x0600c x0600d x0600f }
+      case entry_discharge_reporting
+      when 99
+        fields << 'x0700a'
+      when 01
+        fields << 'x0700c'
+      when 10, 11, 12
+        fields << 'x0700b'
+      end
+      fields
+    end
+
+    def hash_field_values(fields, target=true)
+      field_values = fields.map do |f| 
+        value_field = target ? f : MATCH_MAP[f]
+        "#{f}=#{responses[value_field]}"
+      end
+
+      # binding.pry
+
+      Digest::MD5.hexdigest(field_values.join(','))
+    end
+
+    private
+
+    def filter_value(value)
+      MISSING_ATTRIBUTE_VALUES.include?(value) ? nil : value
+    end
+
+    def twisted_discharge_dates
+      e_date = Date.safe_parse(entry_date)
+      d_date = Date.safe_parse(discharge_date)
+
+      return if e_date.nil? || d_date.nil?
+
+      errors.add('entry_discharge_dates', "Discharge date (a2000)  must be later than or equal to entry date (a1600)") if e_date > d_date
+    end
+
+    def self.disp_abbr
+      "MDS 3.0"
+    end
+
+    def self.find_assessment_data(facility, filter, options={})
+      facility.find_matching_qcli_results(filter, options) 
+    end
+
+    def self.find_non_interviewed_data(facility, filter, options={})
+      return nil
+    end
+
+
+    def self.find_flagged_results_for_qcli_or_subgroup(facility, filter, qcli, subgroup)
+      qcli_ids = Unit.find(7).qclis.collect(&:id).join(",")
+      qcli_results = facility.find_matching_qcli_results(filter, {:qcli_ids => qcli_ids})
+      (subgroup) ? qcli_results.select{|result| result.qcli_id == qcli.id && result.flags_subgroup_numerator?(qcli, subgroup.columns)} : qcli_results.select{|result| result.flags_numerator?(qcli)}
+    end
+
+    def self.initials; self.table_name.split('_')[1..-1].map{|i| i.slice(0,1)}.join; end
+
+    def self.select_latest_assessment_sql(filter, resident_ids_override=nil)
+      resident_ids = resident_ids_override || Resident.resident_ids_in_group_or_sample(filter)
+      resident_sql = (resident_ids.present?) ? "resident_id in (#{resident_ids.join(",")})" : "facility_id = #{filter.facility.id}"
+
+
+      sql=<<-SQL
+        SELECT *
+        FROM (#{order_latest_assessments_for_use(filter.facility.id, filter)}) mds_assmts
+        WHERE #{resident_sql}
+      SQL
+      sql.compact
+    end
+
+    def self.order_latest_assessments_for_use(facility_id, filter)
+
+      date_sql = ""
+      if filter[:mds_date_s].present?
+        date_sql = "AND date_e >= '#{filter['mds_date_s']}' AND date_e <= '#{filter.date_e}'"
+      elsif filter.date_s.present?
+        date_sql = "AND date_e >= '#{filter.date_s}' AND date_e <= '#{filter.date_e}'"
+      end
+
+      #date_sql = (filter.date_s.present?) ? "AND date_e >= '#{filter.date_s}' AND date_e <= '#{filter.date_e}'" : ""
+      #date_sql = (filter[:mds_date_s].present?) ? "AND date_e >= '#{filter['mds_date_s']}' AND date_e <= '#{filter.date_e}'" : ""
+      exclusion_sql = (filter[:mds_exclude_empty_pos_neg].present?) ? " AND NOT(positives LIKE '%[]%' and negatives LIKE '%[]%') " : ""
+      sql=<<-SQL
+          SELECT *
+          FROM
+            (SELECT * FROM qcli_results
+              WHERE facility_id = #{facility_id} #{date_sql} #{exclusion_sql}
+              ORDER BY date_s desc, id desc) mds_assessment_results
+          GROUP BY resident_id, qcli_id
+      SQL
+      sql.compact
+    end
+
+    def has_reference_date?
+      self.reference_date.present?
+    end
+
+    def set_inact_corr_fields
+      self.obra_reason = clean_a0310a
+      self.pps_reason = clean_a0310b
+      self.entry_disch_reporting = [ INACTIVATE_EXISTING_RECORD, MODIFY_EXISTING_RECORD ].include?(type_of_record) ? x0600f.to_i : a0310f
+      self.inact_corr_match = hash_field_values(inact_corr_match_fields(entry_disch_reporting), [ INACTIVATE_EXISTING_RECORD, MODIFY_EXISTING_RECORD ].include?(type_of_record) )
+    end
+
+    def determine_matching_fields
+      self.item_subset_code = itm_sbst_cd
+      self.type_of_record = (version == VERSION1_0 ?  x0100 : a0050).to_i
+      self.previous_reference_date = a2200 if type_of_record == MODIFY_EXISTING_RECORD
+    end
+
+    def determine_reference_date
+      ref_date = entry_assessment? ? self.clean_a1600 : discharge_or_death_assessment? ? self.clean_a2000 : nil
+
+      if inactivation_assessment?
+        ref_date =
+          case  self.x0600f
+            when "99" then self.clean_x0700a
+            when "01" then self.clean_x0700c
+            when "10", "11", "12" then self.clean_x0700b
+            else self.clean_x1100e
+          end
+      end
+
+      ref_date ||= self.clean_a2300
+      ref_date ||= self.clean_z0500b
+
+      self.reference_date = Date.safe_parse(ref_date)
+    end
+
+    def determine_version
+      self.version = self.reference_date >= Date.parse(VERSION1_1_DATE) ? VERSION1_1 : VERSION1_0
+    end
+
+  end
+
+end

data/app/models/services/mds_content.rb

@@ -0,0 +1,55 @@
+module Services
+
+  class MdsContent < ActiveRecord::Base
+    BASE_EXTRACTION_PATH = Rails.root.join('tmp').to_path
+
+    attr_encrypted :content, type: 'binary'
+    belongs_to :uploadable, polymorphic: true
+
+    def each_entry(&block)
+      each_zipped_file('**/*', &block)
+    end
+
+    def each_zipped_file(glob_pattern)
+      write_content_to_file
+      extract_zip_contents
+      Dir.glob(File.join(extracted_files_path, glob_pattern), File::FNM_CASEFOLD).sort.each do |zip_file_path|
+        yield zip_file_path, zip_file_path[extracted_files_path.length + 1..-1]
+      end
+
+    ensure
+      remove_content_file
+      remove_extracted_files
+    end
+
+    def content_path
+      @content_path ||= File.join(BASE_EXTRACTION_PATH, content_file_name)
+    end
+
+    def extracted_files_path
+      @extracted_files_path ||= File.join(BASE_EXTRACTION_PATH, "extracted_files_#{id}")
+    end
+
+    def extract_zip_contents
+      FileUtils.mkdir_p(extracted_files_path) unless File.exists?(extracted_files_path)
+      RecursiveUnzipper.new(content_path).extract_to(extracted_files_path)
+    end
+
+    def write_content_to_file
+      File.open(content_path, "wb") { |f| f.write(content) }
+    end
+
+    def remove_content_file
+      File.unlink(content_path) if File.exists?(content_path)
+    end
+
+    def remove_extracted_files
+      FileUtils.rm_rf(extracted_files_path) if File.exists?(extracted_files_path)
+    end
+
+    def content_file_name
+      @content_file_name ||= "composite_file_#{id}.zip"
+    end
+  end
+
+end

data/app/models/services/mds_pull.rb

@@ -0,0 +1,41 @@
+module Services
+
+  class MdsPull < ServiceImplementation
+
+    #
+    # The async configuration options
+    #
+    ASYNC_SEND_OPTS = {
+      :priority => 100, # Pretty high priority; higher than MDS for sure
+      :time_to_run => 3600 # GIve it an hour
+    }
+
+    #
+    # Main (only public) entry point for the MDS pull service.
+    #
+    def invoke
+
+      service_definition.configured_accounts.enabled.each do | configured_account |
+
+        mds_pull_account = MdsPullAccount.create!(configured_account: configured_account)
+
+        mds_pull_account.ayl_send_opts(:process, ASYNC_SEND_OPTS)
+
+      end
+    end
+
+    def request_baseline(configured_account, provider_id, uploaded_after, uploaded_before)
+      # Do nothing by default; the derived class will handle the necessary details.
+    end
+
+    def identify_providers(configured_account)
+      # Do nothing by default. Only some service providers will provide this feature.
+    end
+
+    def request_assessments(configured_account)
+      # Do nothing by default. Only some service providers will provide this feature.
+    end
+    
+  end
+
+end

data/app/models/services/mds_pull_account.rb

@@ -0,0 +1,192 @@
+module Services
+
+  class MdsPullAccount < ActiveRecord::Base
+    include FileUtils
+    belongs_to :configured_account
+
+    DECAY_TIME = 1200 # 20 minutes
+
+    #
+    # Define all the statuses allowed for this object
+    #
+    STATUS_NEW = 0
+    STATUS_PROVIDERS_IDENTIFIED = 1
+    STATUS_ASSESSMENTS_REQUESTED = 2
+    STATUS_FILES_IDENTIFIED = 3
+    STATUS_FILES_DOWNLOADED = 4
+    STATUS_SUBMITTED_TO_ABAQIS = 5
+    STATUS_FILE_SYSTEM_CLEAN = 6
+    STATUS_REMOTE_CLEAN = 7
+
+    #
+    # Define the order of statuses for this object. The statuses
+    # can proceed ONLY in this order
+    #
+    PROCESSING_STATES = [ STATUS_NEW,
+                          STATUS_PROVIDERS_IDENTIFIED,
+                          STATUS_ASSESSMENTS_REQUESTED,
+                          STATUS_FILES_IDENTIFIED,
+                          STATUS_FILES_DOWNLOADED,
+                          STATUS_SUBMITTED_TO_ABAQIS,
+                          STATUS_FILE_SYSTEM_CLEAN,
+                          STATUS_REMOTE_CLEAN ]
+
+    #
+    # Define the task to perform for each of the valid status, so if
+    # the current status of the object is STATUS_FILES_IDENTIFIED, then
+    # the :download_files method should be invoked. If the method is
+    # sucessful (doesn't raise an exception), the status of the object is automatically
+    # moved to the next valid status (from the list above).
+    # 
+    STATE_MACHINE = {
+      STATUS_NEW                   => :identify_providers, #=> :identify_files_to_pull,
+      STATUS_PROVIDERS_IDENTIFIED  => :request_assessments_to_process,
+      STATUS_ASSESSMENTS_REQUESTED => :identify_files_to_pull,
+      STATUS_FILES_IDENTIFIED      => :download_files,
+      STATUS_FILES_DOWNLOADED      => :submit_files_to_abaqis,
+      STATUS_SUBMITTED_TO_ABAQIS   => :clean_up_file_system,
+      STATUS_FILE_SYSTEM_CLEAN     => :clean_up_remote_server
+    }
+
+    has_one :service_definition, :through => :configured_account
+
+    validates :status, presence: true, inclusion: { in: PROCESSING_STATES }
+    validates :attempt, presence: true
+
+    def process
+      logger.info "Processing configured account: #{configured_account.to_rrepr}"
+
+      # Update the attempt count for the MDS Pull Account record. (how many times have we tried
+      # to process this record?)
+      update_attribute :attempt, attempt + 1
+
+      PROCESSING_STATES.each do | p_status |
+        perform_task p_status
+      end
+
+    rescue Exception => ex
+      logger.error "Error processing configured account: #{self.to_rrepr}: #{ex.message}\n#{ex.backtrace.join("\n")}"
+      if attempt > 1 && (attempt % 3 == 0)
+        ServicesMailer.burying_job(self).deliver
+        raise Ayl::Beanstalk::RequiresJobBury 
+      else
+        ServicesMailer.delaying_job(self).deliver
+        raise Ayl::Beanstalk::RequiresJobDecay.new(DECAY_TIME)
+      end
+    end
+
+    def self.for_service_within_time_frame(service_key, from, to)
+      svc = Service.where(key: service_key).first
+      configured_account_ids = ConfiguredAccount.for_service(svc.id).enabled.pluck(:id)
+
+      self.where("configured_account_id IN (?) AND created_at >= ? AND created_at <= ?",
+                 configured_account_ids, from, to)
+    end
+
+    private
+
+    def identify_providers
+      logger.debug "Identifying configured providers for: #{configured_account.to_rrepr}"
+      service_impl.identify_providers(configured_account)
+    end
+
+    def request_assessments_to_process
+      logger.debug "Requesting assessment to be processed from the third party"
+      service_impl.request_assessments(configured_account)
+    end
+
+    def identify_files_to_pull
+      logger.debug "Identifying files to pull for: #{configured_account.to_rrepr}"
+      list_available_files.each do | file_info |
+        logger.debug "File: #{file_info}"
+        configured_account.available_files.create(request_id: file_info[0], 
+                                                  filename: file_info[1],
+                                                  status: AvailableFile::STATUS_AVAILABLE)
+      end
+    end
+
+    def list_available_files
+      service_impl.list_available_files(configured_account)
+    end
+
+    def download_files
+      logger.debug "Downloading files for: #{configured_account.to_rrepr}"
+
+      # Get the list of available_files that are ready for download
+      files_to_download = configured_account.available_files.available
+
+      # Actually perform the download of the files
+      downloaded_files = perform_file_downloads files_to_download
+      
+      # Update the records
+      if downloaded_files.present?
+        fid_list = downloaded_files.join(",")
+        # Update only the files that were actually downloaded, but do it in a 
+        # single db call.
+        AvailableFile.where("id IN (#{fid_list})").update_all("status = '#{AvailableFile::STATUS_DOWNLOADED}', downloaded_at = '#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}'")
+      end
+    end
+
+    def perform_file_downloads(files_to_download)
+      service_impl.perform_file_downloads(configured_account, files_to_download)
+    end
+
+    def submit_files_to_abaqis
+      logger.debug "Submitting files to abaqis for: #{configured_account.to_rrepr}"
+      mds_push = AbaqisMdsPush.service_impl
+      configured_account.available_files.downloaded.each do | available_file |
+        begin
+          mds_push.push_mds_file(available_file)
+        rescue Exception => ex
+          logger.error "Error pushing file (#{available_file.to_rrepr})) to abaqis: #{ex.message}\n#{ex.backtrace.join("\n")}"
+          available_file.update_attribute :error, "Error pushing file to abaqis: #{ex.message}"
+        end
+      end
+    end
+    
+    def clean_up_file_system
+      logger.debug "Cleaning up file system for: #{configured_account.to_rrepr}"
+      files_to_remove = configured_account.available_files.uploaded
+      files_to_remove.each do | available_file |
+        rm_rf available_file.download_path
+        available_file.update_attribute :status, AvailableFile::STATUS_DELETED
+      end
+
+    end
+
+    def clean_up_remote_server
+      logger.debug "Removing files from remote server for: #{configured_account.to_rrepr}"
+      
+      # Actually remove the files from the server
+      removed_files = perform_cleanup_on_remote_server
+
+      # Update the records
+      if removed_files.present?
+        fid_list = removed_files.join(',')
+        # Update only the files that were actually removed, but do it in a 
+        # single db call.
+        AvailableFile.where("id IN (#{fid_list})").update_all("status = '#{AvailableFile::STATUS_CLEARED}'")
+      end
+    end
+
+    def perform_cleanup_on_remote_server
+      service_impl.perform_cleanup_on_remote_server(configured_account)
+    end
+    
+    def perform_task(p_status)
+      return unless self.status == p_status && STATE_MACHINE[p_status]
+
+      # Invoke the appropriate method for this status
+      send STATE_MACHINE[status]
+
+      # If the processing completes without a hitch, update the status attribute
+      # to the next appropriate status.
+      update_attribute :status, PROCESSING_STATES[PROCESSING_STATES.index(p_status) + 1]
+    end
+
+    def service_impl
+      @service_impl ||= service_definition.service_implementation
+    end
+  end
+
+end

data/app/models/services/mds_push.rb

@@ -0,0 +1,24 @@
+module Services
+
+  class MdsPush < ServiceImplementation
+
+    def invoke
+    end
+
+    def push_mds_file(available_file)
+      raise "Downloaded file does not exist: #{available_file.downloaded_file_path}" unless available_file.downloaded_file_exists?
+      raise "Downloaded file is not valid MDS: #{available_file.downloaded_file_path}" unless available_file.valid_mds_file?
+      submit_to_service(available_file)
+    end
+
+    def self.service_impl
+      nil
+    end
+
+    def submit_to_service(available_file)
+      puts "## Submitting MDS file to service"
+    end
+
+  end
+
+end