openmrs_data_sanitizer 1.0.1

data/app/services/openmrs_data_sanitizer/data_sanitizer.rb

@@ -0,0 +1,414 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'faker'
+require 'fhir_models'
+require 'concurrent-ruby'
+
+# DataSanitizer
+module OpenmrsDataSanitizer
+  # Error < StandardError; end
+
+  # rubocop:disable Metrics/ClassLength
+  # AWS DataSanitizer
+  class DataSanitizer
+    attr_accessor :fhir_encounters, :fhir_observations, :regimen_combinations,
+                  :concept_names, :faker_names, :sites, :patients_json,
+                  :fhir_medication_dispense
+
+    def initialize(sites: [])
+      @fhir_encounters = Concurrent::Array.new
+      @fhir_observations = Concurrent::Array.new
+      @fhir_medication_dispense = Concurrent::Array.new
+      @concept_names = Concurrent::Hash.new
+      @faker_names = Concurrent::Hash.new
+      @patients_json = Concurrent::Array.new
+      @regimen_combinations = {}
+      @sites = sites
+      load_regimen_combinations
+    end
+
+    def start
+      start_time = Time.now
+      Rails.logger.info("DataSanitizer started at #{start_time}")
+      clear_directory
+      process_data_request
+      end_time = Time.now
+      time_taken = (end_time - start_time) / 60
+      Rails.logger.info("DataSanitizer completed at #{end_time} and it took #{time_taken} minutes")
+      { start_time:, end_time:, time_in_minutes: (end_time - start_time) / 60 }
+    end
+
+    private
+
+    FILE_MUTEX = Mutex.new
+
+    # ===================================
+    # REGION: DATA PROCESSING
+    # ===================================
+
+    # rubocop:disable Metrics/MethodLength
+    def process_data_request
+      queue = Queue.new
+      sites.each { |loc| queue << loc }
+
+      threads = Array.new(10) do
+        Thread.new do
+          until queue.empty?
+            loc = begin
+              queue.pop(true)
+            rescue StandardError
+              nil
+            end
+            next unless loc
+
+            process_location(loc)
+          end
+        end
+      end
+
+      threads.each(&:join)
+      write_data_to_files
+    end
+
+    def process_location(loc)
+      ActiveRecord::Base.connection_pool.with_connection do
+        site = ::SchemaService.new.by_site(loc)&.first
+        return unless site
+
+        site_name = site['facility']
+        schema = site['schema']
+
+        OpenmrsDataSanitizer::DataAggregator.new(schema:).aggregate
+
+        process_patients(schema, site_name)
+      rescue StandardError => e
+        Rails.logger.info("Error processing location #{loc}: #{e.message}")
+        Rails.logger.info(e.backtrace.join("\n"))
+      end
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def process_patients(schema, site_name)
+      patients = fetch_patients(schema:)
+      patients_ids = patients.map { |patient| patient['person_id'].to_i }
+      patients.map { |patient| build_patient_json(patient, site_name) }
+
+      visits = fetch_visits(patients_ids:, schema:)
+      process_visits(visits:)
+      process_medication(schema:)
+    end
+
+    def write_data_to_files
+      write_to_file(patients_json.to_json, 'patients')
+      write_to_file(fhir_encounters.to_json, 'visits')
+      write_to_file(fhir_observations.to_json, 'observations')
+      write_to_file(fhir_medication_dispense.to_json, 'medications')
+    end
+
+    # ===================================
+    #  REGION: REGIMEN COMBINATIONS
+    # ===================================
+
+    # rubocop:disable Metrics/MethodLength
+    def load_regimen_combinations
+      combinations = ActiveRecord::Base.connection.select_all <<~SQL
+        SELECT
+          moh_regimen_combination.regimen_combination_id,
+          GROUP_CONCAT(DISTINCT moh_regimen_combination_drug.drug_id
+                       ORDER BY moh_regimen_combination_drug.drug_id
+                       ASC SEPARATOR ',') AS drugs,
+          moh_regimen_name.name
+        FROM moh_regimen_combination_drug
+        INNER JOIN (
+          SELECT regimen_combination_id
+          FROM moh_regimen_combination_drug
+        ) AS potential_combinations ON potential_combinations.regimen_combination_id = moh_regimen_combination_drug.regimen_combination_id
+        INNER JOIN moh_regimen_combination ON moh_regimen_combination.regimen_combination_id = moh_regimen_combination_drug.regimen_combination_id
+        INNER JOIN moh_regimen_name ON moh_regimen_name.regimen_name_id = moh_regimen_combination.regimen_name_id
+        GROUP BY moh_regimen_combination.regimen_combination_id;
+      SQL
+
+      (combinations || []).each do |combination|
+        regimen_combinations[combination['drugs']] = combination['name']
+      end
+    end
+
+    # ===================================
+    #  REGION: AWS FHIR PATIENT
+    # ===================================
+
+    def fetch_patients(schema:)
+      ActiveRecord::Base.connection.select_all <<~SQL
+        SELECT
+          p.person_id, given_name, middle_name, family_name, gender, birthdate,
+          phone.value phone_number, village.value village_name
+        FROM #{schema}.person p
+        INNER JOIN #{schema}.patient ON patient.patient_id = p.person_id AND p.voided = 0
+        INNER JOIN #{schema}.patient_program pp ON pp.patient_id = p.person_id AND pp.voided = 0 AND pp.program_id = 1
+        LEFT JOIN #{schema}.person_name n ON n.person_id = p.person_id AND n.voided = 0
+        LEFT JOIN #{schema}.person_attribute phone ON phone.person_id = p.person_id AND phone.voided = 0 AND phone.person_attribute_type_id = 12
+        LEFT JOIN #{schema}.person_attribute village ON village.person_id = p.person_id AND village.voided = 0 AND village.person_attribute_type_id = 19;
+      SQL
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def build_patient_json(patient, site_name)
+      patient_id = patient['person_id'].to_i
+      village_name = get_or_generate_faker_value(patient['village_name'], Faker::Address, :community)
+      cell_number = get_or_generate_faker_value(patient['phone_number'], Faker::PhoneNumber, :phone_number)
+      family_name = get_or_generate_faker_value(patient['family_name'], Faker::Name, :last_name)
+      given_name = get_or_generate_faker_value(patient['given_name'], Faker::Name, :first_name)
+      middle_name = get_or_generate_faker_value(patient['middle_name'], Faker::Name, :middle_name)
+
+      patients_json << FHIR::Patient.new(
+        id: patient_id,
+        identifier: [{ system: "#{site_name}_#{patient_id}", value: patient_id }],
+        name: [{ use: 'official', family: family_name, given: [given_name, middle_name] }],
+        gender: patient['gender'].match(/f/i) ? 'female' : 'male',
+        birthDate: patient['birthdate'],
+        address: [{ use: 'home', village: village_name }],
+        telecom: [{ system: 'phone', value: cell_number, use: 'mobile' }]
+      )
+    end
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+    def fetch_visits(patients_ids:, schema:)
+      ActiveRecord::Base.connection.select_all <<~SQL
+        SELECT * FROM #{schema}.data_sanitizer_visits
+        WHERE patient_id IN(#{patients_ids.join(',')})
+      SQL
+    end
+
+    def process_visits(visits:)
+      (visits || []).map do |visit|
+        arvs_given = visit['drugs_dispensed'].to_s.split(',').map(&:to_i).sort
+        regimen = arvs_given.blank? ? 'N/A' : (regimen_combinations[arvs_given.join(',').squish] || 'Unknown')
+        aws_fhir_encounter(visit:, regimen:)
+      end
+    end
+
+    # ===================================
+    #  REGION: AWS FHIR ENCOUNTER
+    # ===================================
+
+    def aws_fhir_encounter(visit:, regimen:)
+      patient_id = visit['patient_id'].to_i
+      fhir_encounters << create_fhir_encounter(patient_id, visit['visit_date'])
+
+      observations = process_indicators(visit, regimen)
+      create_fhir_observations(observations, patient_id, visit['visit_date'])
+    end
+
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    def create_fhir_encounter(patient_id, visit_date)
+      FHIR::Encounter.new(
+        id: Digest::SHA256.hexdigest(patient_id.to_s),
+        status: 'finished',
+        class: {
+          system: 'EGPAF Malawi EMR',
+          code: 'PVE',
+          display: 'Patient visit encounter'
+        },
+        subject: {
+          reference: "Patient/#{patient_id}"
+        },
+        period: {
+          start: visit_date,
+          end: visit_date
+        }
+      )
+    end
+
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/BlockLength
+    def process_indicators(visit, regimen)
+      visit.keys.map do |indicator|
+        next if indicator.match(/patient_id|visit_date|outcome_concept_id|outcome_end_date|other/i)
+
+        case indicator
+        when /weight|height/i
+          {
+            code: (indicator.match(/weight/i) ? 5089 : 5090),
+            display: indicator.titleize,
+            value: visit[indicator],
+            unit: (indicator.match(/weight/i) ? 'Kg' : 'cm')
+          }
+        when /adherence/i
+          {
+            code: get_concept_name(indicator.gsub('_', ' ')).concept_id,
+            display: indicator.titleize,
+            value: (visit[indicator].blank? ? visit[indicator] : visit[indicator].to_s.gsub('%', '').to_f),
+            unit: '%'
+          }
+        when /drugs_dispensed/i
+          {
+            code: 6882,
+            display: 'ARV regimen type',
+            value: regimen
+          }
+        else
+          {
+            code: get_concept_name(indicator.gsub('_', ' ')).concept_id,
+            display: indicator.titleize,
+            value: visit[indicator]
+          }
+        end
+      end.compact
+    end
+
+    # rubocop:disable Metrics/PerceivedComplexity
+    def create_fhir_observations(observations, patient_id, visit_date)
+      observations.each do |obs|
+        next if obs.blank? || obs[:code].blank? || obs[:value].blank?
+
+        fhir_observations << FHIR::Observation.new(
+          status: 'final',
+          code: {
+            coding: [
+              {
+                system: 'EGPAF Malawi EMR',
+                code: obs[:code],
+                display: obs[:display]
+              }
+            ]
+          },
+          subject: {
+            reference: "Patient/#{patient_id}"
+          },
+          encounter: {
+            reference: "Encounter/#{fhir_encounters.last.id}"
+          },
+          effectiveDateTime: visit_date
+        )
+
+        if obs[:unit]
+          fhir_observations.last.valueQuantity = {
+            value: obs[:value],
+            unit: obs[:unit],
+            system: 'EGPAF Malawi EMR',
+            code: obs[:code]
+          }
+        else
+          fhir_observations.last.valueCodeableConcept = {
+            coding: [
+              {
+                system: 'EGPAF Malawi EMR',
+                code: obs[:code],
+                display: (if obs[:value] == '1065'
+                            'Yes'
+                          else
+                            obs[:value] == '1066' ? 'No' : obs[:value]
+                          end)
+              }
+            ]
+          }
+        end
+      end
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength, Metrics/PerceivedComplexity
+
+    # ===================================
+    # REGION: AWS FHIR MEDICATIONS
+    # ===================================
+
+    def process_medication(schema:)
+      orders = fetch_medication(schema:)
+      create_fhir_medications(orders:)
+    end
+
+    def fetch_medication(schema:)
+      ActiveRecord::Base.connection.select_all <<~SQL
+        SELECT * FROM #{schema}.data_sanitizer_arv_orders
+      SQL
+    end
+
+    def create_fhir_medications(orders:)
+      fhir_medication_dispense << (orders || []).map do |order|
+        patient_id = order['patient_id'].to_i
+        order_id = order['order_id'].to_i
+        encounter_id = order['encounter_id'].to_i
+        FHIR::MedicationDispense.new(
+          identifier: {
+            value: order_id
+          },
+          status: 'completed',
+          patient: {
+            reference: encounter_id
+          },
+          medication: {
+            code: {
+              text: order['drug_name'],
+              coding: [
+                {
+                  system: 'EGPAF Malawi EMR',
+                  code: order['drug_id'],
+                  display: order['drug_name']
+                }
+              ]
+            }
+          },
+          dispensed: {
+            quantity: {
+              value: order['quantity'].to_f,
+              unit: order['units']
+            },
+            type: {
+              text: order['units']
+            },
+            dateTime: order['start_date']
+          },
+          dosageInstruction: [
+            {
+              text: order['instructions']
+            }
+          ],
+          dispense: {
+            validityPeriod: {
+              start: order['start_date'],
+              end: order['runout_date']
+            }
+          }
+        )
+      end
+    end
+
+    # ===================================
+    #  Helper methods
+    # ===================================
+
+    def get_or_generate_faker_value(attribute, faker_module, faker_method)
+      return '' if attribute.blank?
+
+      @faker_names[attribute] ||= faker_module.send(faker_method)
+    end
+
+    def get_concept_name(name)
+      concept_names[name] ||= ConceptName.find_by_name(name)
+    end
+
+    def clear_directory
+      dir_path = Rails.root.join('log/openmrs_data_sanitizer')
+      return unless Dir.exist?(dir_path)
+
+      FileUtils.rm_rf(dir_path)
+    end
+
+    def write_to_file(json_file, file_name)
+      dir_path = Rails.root.join('log/openmrs_data_sanitizer')
+      file_path = dir_path.join("#{file_name}.json")
+
+      # Ensure the directory exists
+      Dir.mkdir(dir_path) unless Dir.exist?(dir_path)
+
+      # Use the mutex to handle concurrent writes
+      FILE_MUTEX.synchronize do
+        File.open(file_path, 'a') do |out_file|
+          out_file.puts(json_file)
+          puts "Created file: #{file_path}"
+        end
+      end
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/openmrs_data_sanitizer/engine.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module OpenmrsDataSanitizer
+  # Engine class
+  class Engine < ::Rails::Engine
+    isolate_namespace OpenmrsDataSanitizer
+    config.generators.api_only = true
+
+    config.generators do |g|
+      g.test_framework :rspec
+    end
+  end
+end

data/lib/openmrs_data_sanitizer/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module OpenmrsDataSanitizer
+  VERSION = '1.0.1'
+end

data/lib/openmrs_data_sanitizer.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# Path: lib/malawi_hiv_program_reports.rb
+require 'openmrs_data_sanitizer/version'
+require 'openmrs_data_sanitizer/engine'
+
+# main module of the gem
+module OpenmrsDataSanitizer
+  def self.hi
+    puts 'Welcome to MalawiHivProgramReports!'
+  end
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: openmrs_data_sanitizer
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Mwatha
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-08-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.20'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.20'
+- !ruby/object:Gem::Dependency
+  name: fhir_models
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.6
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.1.3
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.1.3.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.1.3
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.1.3.4
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+description: This gem will mask all patient identifyable data before exporting to
+  AWS FHIR API
+email:
+- mwathabwanali@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- app/services/openmrs_data_sanitizer/data_aggregator.rb
+- app/services/openmrs_data_sanitizer/data_sanitizer.rb
+- lib/openmrs_data_sanitizer.rb
+- lib/openmrs_data_sanitizer/engine.rb
+- lib/openmrs_data_sanitizer/version.rb
+homepage: https://github.com/egpaf-global/openmrs_data_sanitizer
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/egpaf-global/openmrs_data_sanitizer
+  source_code_uri: https://github.com/egpaf-global/openmrs_data_sanitizer
+  changelog_uri: https://github.com/egpaf-global/openmrs_data_sanitizer/blob/master/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.5
+signing_key:
+specification_version: 4
+summary: This gem will mask all patient identifyable data before exporting to AWS
+  FHIR API
+test_files: []