malawi_hiv_program_reports 1.1.16 → 1.1.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 394a402d0b5e0c9b210bbf5feff0959854bb2b8d1322ecd97b07107201dcd694
-  data.tar.gz: 51615fcd686c06088e928b6b10e1bba26a754825e9c3d891e0e7c43c00647098
+  metadata.gz: 22a3e944fcb8f4a9e0d862b50321ead38c607a3488c002096735853b06922be5
+  data.tar.gz: fb58caba60b6795e915efaadd08fea81e2817129ef500843da95ba48047028aa
 SHA512:
-  metadata.gz: aa30a38678f9f6dbeded4d8b3fe6ae920a46c6bea63937906f330cdf50b2cbc1ca6ae95edcf004caa2b4ca5501c6dbf627210f2f496dd49e1ba3964815e3a28f
-  data.tar.gz: cccee12cb7e4e5d9e5e685230b63309f93ad6db7199de3ae244d461a95e0f825f71a0d82f9b01a535636ad2c45fffc617dfcc68460d1f9508853c059ee71be9e
+  metadata.gz: 91ae6c317a44e67d502cc97f3d3d72d54ace89d8def8326f8fc2ea7901e1b4693c0754084b33a1bf9c9d32626dd840b72df1e1c56ee558547ed0ff5056756403
+  data.tar.gz: 8d52d72581ab43cc207e34bbb20ecf62d942cd104c854e0890c4b1712346d64a609907cd447c9f912298be40b78d0affd36e4be1a8ef123a86f7fd749258a83b

data/app/services/malawi_hiv_program_reports/cohort/disaggregated.rb

@@ -29,7 +29,7 @@ module MalawiHivProgramReports
         rebuild_report if rebuild
         process_initialization
         process_data
-        flatten_and_sort_data
+        flatten_the_report
       rescue StandardError => e
         Rails.logger.info("Error processing location #{location}: #{e.message}")
         Rails.logger.info(e.backtrace.join("\n"))
@@ -38,12 +38,12 @@ module MalawiHivProgramReports
 
       private
 
-      GENDER = %w[M F].freeze
-      AGGREGATE_GENDER_ROWS = %w[M FP FNP FBf].freeze
+      GENDER = %w[Male Female].freeze
+      AGGREGATE_GENDER_ROWS = %w[Male FP FNP FBf].freeze
 
       def process_initialization
         init_report
-        init_aggregate_rows if aggregation
+        init_aggregate_rows
       end
 
       def init_report
@@ -85,6 +85,7 @@ module MalawiHivProgramReports
           regimen = data['regimen'] || 'unknown'
           regimen = 'unknown' unless COHORT_REGIMENS.include?(regimen)
           patient_id = data['patient_id']
+          maternal_status = data['maternal_status']
           # we need to handle regimes that only have one P to become PP. Otherwise if it is already PP or PA we leave
           # it as is. Regimens are in this format NUMBERLETTERS
           regimen = regimen.gsub(/(\d+[A-Za-z]*P)\z/, '\1P') if regimen.match?(/\A\d+[A-Za-z]*[^P]P\z/)
@@ -92,63 +93,65 @@ module MalawiHivProgramReports
           report[age_group.to_s][gender.to_s][regimen.to_s] << patient_id
           report[age_group.to_s][gender.to_s]['tx_curr'] << patient_id
           report[age_group.to_s][gender.to_s]['total'] << patient_id
-          process_aggregate_rows(gender:, regimen:, patient_id:) if aggregation
+          process_aggregate_rows(gender:, regimen:, patient_id:, maternal_status:)
         end
       end
 
-      def process_maternal_data
-        result = MalawiHivProgramReports::Pepfar::ViralLoadCoverage2.new(start_date:, end_date:, location:)
-                                                                    .vl_maternal_status(maternal.keys)
-        # result comes in this form: { FP: [], FBf: [] }
-        # we need to loop through the keys
-        result.each_key do |key|
-          result[key].each do |patient_id|
-            report['All'][key.to_s]['tx_curr'] << patient_id
-            report['All'][key.to_s][maternal[patient_id].to_s] << patient_id
-            report['All'][key.to_s]['total'] << patient_id
-          end
-        end
-      end
-      # rubocop:enable Metrics/AbcSize
-
-      def process_aggregate_rows(gender:, regimen:, patient_id:)
+      def process_aggregate_rows(gender:, regimen:, patient_id:, maternal_status: nil)
         if gender == 'M'
           report['All']['M']['tx_curr'] << patient_id
           report['All']['M'][regimen.to_s] << patient_id
           report['All']['M']['total'] << patient_id
-        else
-          maternal[patient_id] = regimen
+          return
         end
-      end
 
-      # we need to flatten the data
-      def flatten_data
-        flat_data = []
-        report.each do |age_group, gender_data|
-          gender_data.each do |gender, regimen_data|
-            flat_data << {
-              'age_group' => age_group,
-              'gender' => gender,
-              **regimen_data
-            }
-          end
+        case maternal_status
+        when 'FP'
+          report['All']['FP']['tx_curr'] << patient_id
+          report['All']['FP'][regimen.to_s] << patient_id
+          report['All']['FP']['total'] << patient_id
+        when 'FBf'
+          report['All']['FBf']['tx_curr'] << patient_id
+          report['All']['FBf'][regimen.to_s] << patient_id
+          report['All']['FBf']['total'] << patient_id
+        else
+          report['All']['FNP']['tx_curr'] << patient_id
+          report['All']['FNP'][regimen.to_s] << patient_id
+          report['All']['FNP']['total'] << patient_id
         end
-        flat_data
       end
+      # rubocop:enable Metrics/AbcSize
 
-      def flatten_and_sort_data
-        flat_data = flatten_data
+      def flatten_the_report
+        result = []
+        @report.each do |age_group, age_group_report|
+          next if age_group == 'Unknown'
 
-        age_group_order = ['Unknown', '<1 year', '1-4 years', '5-9 years', '10-14 years', '15-19 years',
-                           '20-24 years', '25-29 years', '30-34 years', '35-39 years', '40-44 years', '45-49 years',
-                           '50-54 years', '55-59 years', '60-64 years', '65-69 years', '70-74 years', '75-79 years',
-                           '80-84 years', '85-89 years', '90 plus years', 'All']
-        gender_order = %w[F M FP FNP FBF]
+          age_group_report.each_key do |gender|
+            next if gender == 'Unknown'
 
-        flat_data.sort_by do |row|
-          [age_group_order.index(row['age_group']) || Float::INFINITY,
-           gender_order.index(row['gender']) || Float::INFINITY]
+            result << process_age_group_report(age_group, gender, age_group_report[gender])
+          end
         end
+
+        new_group = pepfar_age_groups.map { |age_group| age_group }
+        gender_scores = { 'Female' => 0, 'Male' => 1, 'FNP' => 3, 'FP' => 2, 'FBf' => 4 }
+        result_scores = result.sort_by do |item|
+          gender_score = gender_scores[item[:gender]] || 999
+          age_group_score = new_group.index(item[:age_group]) || 999
+          [gender_score, age_group_score]
+        end
+        # remove all unknown age groups
+        result_scores.reject { |item| item[:age_group].match?(/unknown/i) }
+      end
+
+      def process_age_group_report(age_group, gender, age_group_report)
+        {
+          age_group:,
+          gender:,
+          # spread the age group report
+          **age_group_report
+        }
       end
 
       # rubocop:disable Metrics/MethodLength
@@ -158,16 +161,23 @@ module MalawiHivProgramReports
               prescriptions.patient_id,
               COALESCE(regimens.name, 'unknown') AS regimen,
               prescriptions.age_group,
-              prescriptions.gender
+              CASE prescriptions.gender
+                WHEN 'M' THEN 'Male'
+                WHEN 'F' THEN 'Female'
+                ELSE 'Unknown'
+              END gender,
+              prescriptions.maternal_status
           FROM (
               SELECT
                   tcm.patient_id,
                   GROUP_CONCAT(DISTINCT(tcm.drug_id) ORDER BY tcm.drug_id ASC) AS drugs,
                   disaggregated_age_group(date(earliest_start_date.birthdate), date('#{end_date}')) AS age_group,
+                  c.maternal_status,
                   earliest_start_date.gender
               FROM cdr_temp_current_medication #{current_partition} tcm
               INNER JOIN cdr_temp_patient_outcomes #{current_partition} AS outcomes ON outcomes.patient_id = tcm.patient_id AND outcomes.cum_outcome = 'On antiretrovirals'
               INNER JOIN cdr_temp_cohort_members #{current_partition} AS earliest_start_date ON earliest_start_date.patient_id = tcm.patient_id AND earliest_start_date.gender IN ('M','F')
+              LEFT JOIN cdr_temp_maternal_status #{current_partition} c ON c.patient_id = tcm.patient_id
               GROUP BY tcm.patient_id
           ) AS prescriptions
           LEFT JOIN (

data/app/services/malawi_hiv_program_reports/moh/cumulative_cohort.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+# 
+require 'redis'
 
 module MalawiHivProgramReports
   module Moh
@@ -21,10 +23,7 @@ module MalawiHivProgramReports
         @definition = kwargs[:definition] || 'pepfar'
         @iteration = Concurrent::AtomicFixnum.new(0)
         @start_time = Time.now
-        @redis = Redis.new
-        @redis.set('cumulative_progress', 0)
-        @redis.set('cumulative_status', 'running')
-        @redis.set('cumulative_time_taken', 0)
+        prepare_redis
         definition = @definition.downcase
         raise ArgumentError, "Invalid outcomes definition: #{definition}" unless %w[moh pepfar].include?(definition)
       end
@@ -59,7 +58,7 @@ module MalawiHivProgramReports
         queue = Queue.new
         locations.each { |loc| queue << loc }
 
-        threads = Array.new(65) do
+        threads = Array.new(configured_threads) do
           Thread.new do
             until queue.empty?
               loc = begin
@@ -86,6 +85,22 @@ module MalawiHivProgramReports
       end
       # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
+      def configured_threads
+        threads = ::Utils::ConfigReader.application_read('max_threads')&.to_i
+        Rails::logger.info("Max threads: #{threads}")
+        threads.positive? ? threads : 60
+      rescue StandardError => e
+        Rails.logger.info("Error reading max threads: #{e.message}")
+        60
+      end
+
+      def prepare_redis
+        @redis = Redis.new
+        @redis.set('cumulative_progress', 0)
+        @redis.set('cumulative_status', 'running')
+        @redis.set('cumulative_time_taken', 0)
+      end
+
       # update progress in redis and channel
       def update_progress(current_iteration, total)
         percentage = (current_iteration.to_f / total * 100).round(2)
@@ -161,13 +176,22 @@ module MalawiHivProgramReports
         outcome = MalawiHivProgramReports::Moh::CumulativeOutcome.new(end_date:, location:, definition:, rebuild:,
                                                                       start_date:)
         rebuild ? outcome.find_report : outcome.update_outcomes_by_definition
+        other_dependables(location)
         end_time = Time.now
         time_taken = ((end_time - start_time) / 60).round(2)
         save_completed_site(location:, time_taken:)
       end
 
+      def other_dependables(loc)
+        Pepfar::TxTb.new(start_date:, end_date:, location: loc, occupation: nil).process_tb_data
+        Pepfar::MaternalStatus.new(start_date:, end_date:, location: loc).process_data
+      end
+
       # rubocop:disable Metrics/CyclomaticComplexity
       def prepare_tables
+        create_cdr_temp_maternal_status unless check_if_table_exists('cdr_temp_maternal_status')
+        create_cdr_tb_confirmed_and_on_treatment unless check_if_table_exists('cdr_tb_confirmed_and_on_treatment')
+        create_tb_screened unless check_if_table_exists('cdr_tb_screened')
         create_cdr_other_patient_types unless check_if_table_exists('cdr_other_patient_types')
         create_temp_potential_cohort_members_table unless check_if_table_exists('cdr_temp_potential_cohort_members')
         create_min_drug_orders_table unless check_if_table_exists('cdr_temp_min_drug_orders')
@@ -193,6 +217,18 @@ module MalawiHivProgramReports
         SQL
       end
 
+      def create_cdr_temp_maternal_status
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE TABLE IF NOT EXISTS cdr_temp_maternal_status (
+              patient_id INT(11) NOT NULL,
+              site_id INT(11) NOT NULL,
+              maternal_status VARCHAR(5) DEFAULT NULL,
+              PRIMARY KEY (patient_id, site_id)
+          ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+          PARTITION BY LIST(site_id) (#{partition_by_site})
+        SQL
+      end
+
       def create_cdr_temp_cohort_status
         ActiveRecord::Base.connection.execute <<~SQL
           CREATE TABLE IF NOT EXISTS cdr_temp_cohort_status (
@@ -303,6 +339,40 @@ module MalawiHivProgramReports
         SQL
       end
 
+      def create_cdr_tb_confirmed_and_on_treatment
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE TABLE IF NOT EXISTS cdr_tb_confirmed_and_on_treatment (
+            patient_id INT(11) NOT NULL,
+            site_id INT(11) NOT NULL,
+            gender VARCHAR(50) DEFAULT NULL,
+            age_group VARCHAR(50) DEFAULT NULL,
+            tb_confirmed_date DATE DEFAULT NULL,
+            has_tb_confirmed_date BOOLEAN DEFAULT NULL,
+            enrollment_date DATE DEFAULT NULL,
+            prev_reading DATE DEFAULT NULL,
+            PRIMARY KEY (patient_id, site_id)
+          ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+          PARTITION BY LIST(site_id) (#{partition_by_site})
+        SQL
+      end
+
+      def create_tb_screened
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE TABLE IF NOT EXISTS cdr_tb_screened (
+            patient_id INT(11) NOT NULL,
+            site_id INT(11) NOT NULL,
+            gender VARCHAR(255) NULL,
+            screened_date DATE NULL,
+            enrollment_date DATE NULL,
+            age_group VARCHAR(255) NULL,
+            tb_status VARCHAR(255) NULL,
+            screening_methods VARCHAR(255) NULL,
+            PRIMARY KEY (patient_id, site_id)
+          ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+          PARTITION BY LIST(site_id) (#{partition_by_site})
+        SQL
+      end
+
       def create_cdr_other_patient_types
         ActiveRecord::Base.connection.execute <<~SQL
           CREATE TABLE IF NOT EXISTS cdr_other_patient_types (
@@ -485,6 +555,8 @@ module MalawiHivProgramReports
       def clear_tables
         # if locations is empty then we truncating otherwise we clear the locations
         if locations.empty?
+          ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_tb_confirmed_and_on_treatment')
+          ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_tb_screened')
           ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_other_patient_types')
           ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_temp_potential_cohort_members')
           ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_temp_min_drug_orders')
@@ -493,6 +565,8 @@ module MalawiHivProgramReports
           ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_temp_external_clients')
           ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_reason_for_starting_art')
         else
+          ActiveRecord::Base.connection.execute("DELETE FROM cdr_tb_confirmed_and_on_treatment WHERE site_id IN (#{locations.join(',')})")
+          ActiveRecord::Base.connection.execute("DELETE FROM cdr_tb_screened WHERE site_id IN (#{locations.join(',')})")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_other_patient_types WHERE site_id IN (#{locations.join(',')})")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_potential_cohort_members WHERE site_id IN (#{locations.join(',')})")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_min_drug_orders WHERE site_id IN (#{locations.join(',')})")

data/app/services/malawi_hiv_program_reports/pepfar/maternal_status.rb

@@ -22,7 +22,109 @@ module MalawiHivProgramReports
       end
 
       def find_report
-        MalawiHivProgramReports::Pepfar::ViralLoadCoverage2.new(tx_curr_definition: 'pepfar', start_date: @start_date, end_date: @end_date, location: @location).vl_maternal_status(@patient_ids)
+        vl_maternal_status
+      end
+      
+      def process_data
+        clear_maternal_status
+        load_pregnant_women
+        load_breast_feeding
+      end
+
+      private 
+
+      def vl_maternal_status(patient_list)
+        return { FP: [], FBf: [] } if patient_list.blank?
+
+        pregnant = pregnant_women(patient_list).map { |woman| woman['person_id'].to_i }
+        return { FP: pregnant, FBf: [] } if (patient_list - pregnant).blank?
+
+        feeding = breast_feeding(patient_list - pregnant).map { |woman| woman['person_id'].to_i }
+
+        {
+          FP: pregnant,
+          FBf: feeding
+        }
+      end
+
+      def pregnant_women(patient_list)
+        ActiveRecord::Base.connection.execute <<~SQL
+          SELECT person_id
+          FROM cdr_temp_maternal_status #{current_partition}
+          WHERE maternal_status = 'FP' AND patient_od IN (#{patient_list.join(',')})
+        SQL
+      end
+
+      def breast_feeding(patient_list)
+        ActiveRecord::Base.connection.execute <<~SQL
+          SELECT person_id
+          FROM cdr_temp_maternal_status #{current_partition}
+          WHERE maternal_status = 'FBf' AND patient_id IN (#{patient_list.join(',')})
+        SQL
+      end
+
+      def load_pregnant_women
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO cdr_temp_maternal_status #{current_partition} (patient_id, site_id, maternal_status)
+          SELECT o.person_id, #{location}, 'FP' as maternal_status
+          FROM obs #{current_partition} o
+          INNER JOIN cdr_temp_cohort_members #{current_partition} c ON c.patient_id = o.person_id AND c.gender = 'F'
+          LEFT JOIN obs #{current_partition} a ON a.person_id = o.person_id AND a.obs_datetime > o.obs_datetime AND a.concept_id IN (#{pregnant_concepts.to_sql}) AND a.voided = 0
+          AND a.obs_datetime >= DATE(#{ActiveRecord::Base.connection.quote(start_date)}) AND a.obs_datetime < DATE(#{ActiveRecord::Base.connection.quote(end_date)}) + INTERVAL 1 DAY
+          WHERE a.obs_id is null
+            AND o.obs_datetime >= DATE(#{ActiveRecord::Base.connection.quote(start_date)})
+            AND o.obs_datetime < DATE(#{ActiveRecord::Base.connection.quote(end_date)}) + INTERVAL 1 DAY
+            AND o.voided = 0
+            AND o.concept_id in (#{pregnant_concepts.to_sql})
+            AND o.value_coded IN (#{yes_concepts.join(',')})
+          GROUP BY o.person_id
+        SQL
+      end
+
+      def load_breast_feeding
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO cdr_temp_maternal_status #{current_partition} (patient_id, site_id, maternal_status)
+          SELECT o.person_id, #{location}, 'FBf' as maternal_status
+          FROM obs #{current_partition} o
+          INNER JOIN cdr_temp_cohort_members #{current_partition} c ON c.patient_id = o.person_id AND c.gender = 'F'
+          LEFT JOIN obs #{current_partition} a ON a.person_id = o.person_id AND a.obs_datetime > o.obs_datetime AND a.concept_id IN (#{breast_feeding_concepts.to_sql}) AND a.voided = 0
+          AND a.obs_datetime >= DATE(#{ActiveRecord::Base.connection.quote(start_date)}) AND a.obs_datetime < DATE(#{ActiveRecord::Base.connection.quote(end_date)}) + INTERVAL 1 DAY
+          WHERE a.obs_id is null
+            AND o.obs_datetime >= DATE(#{ActiveRecord::Base.connection.quote(start_date)})
+            AND o.obs_datetime < DATE(#{ActiveRecord::Base.connection.quote(end_date)}) + INTERVAL 1 DAY
+            AND o.voided = 0
+            AND o.concept_id IN (#{breast_feeding_concepts.to_sql})
+            AND o.value_coded IN (#{yes_concepts.join(',')})
+            AND o.person_id NOT IN (SELECT c.patient_id FROM cdr_temp_maternal_status #{current_partition} c)
+          GROUP BY o.person_id
+        SQL
+      end
+
+      def clear_maternal_status
+        ActiveRecord::Base.connection.execute <<~SQL
+          DELETE FROM cdr_temp_maternal_status where site_id = #{location}
+        SQL
+      end
+      
+      def yes_concepts
+        @yes_concepts ||= ::ConceptName.where(name: 'Yes').select(:concept_id).map do |record|
+          record['concept_id'].to_i
+        end
+      end
+
+      def pregnant_concepts
+        @pregnant_concepts ||= ::ConceptName.where(name: ['Is patient pregnant?', 'patient pregnant'])
+                                            .select(:concept_id)
+      end
+
+      def breast_feeding_concepts
+        @breast_feeding_concepts ||= ::ConceptName.where(name: ['Breast feeding?', 'Breast feeding', 'Breastfeeding'])
+                                                  .select(:concept_id)
+      end
+
+      def encounter_types
+        @encounter_types ||= ::EncounterType.where(name: ['HIV CLINIC CONSULTATION', 'HIV STAGING'])
+                                            .select(:encounter_type_id)
       end
     end
   end