malawi_hiv_program_reports 1.1.16 → 1.1.17

data/app/services/malawi_hiv_program_reports/pepfar/tx_ml.rb

@@ -3,7 +3,7 @@
 module MalawiHivProgramReports
   module Pepfar
     class TxMl
-      attr_reader :start_date, :end_date, :location, :rebuild
+      attr_reader :start_date, :end_date, :location, :report, :rebuild
 
       include Utils
       include MalawiHivProgramReports::Utils::CommonSqlQueryUtils
@@ -17,56 +17,140 @@ module MalawiHivProgramReports
       end
 
       def find_report
-        if rebuild
-          cohort = MalawiHivProgramReports::Moh::CumulativeCohort.new(start_date: start_date.to_date, end_date: end_date.to_date, definition: 'pepfar',
-                                                                      locations: location.to_s, rebuild: true.to_s)
-          cohort.find_report
-        end
+        rebuild_outcomes if rebuild
+        init_report
+        addittional_groups
         process_data
+        flatten_the_report
       end
 
       private
 
+      def rebuild_outcomes
+        cohort = MalawiHivProgramReports::Moh::CumulativeCohort.new(start_date: start_date.to_date, end_date: end_date.to_date, definition: 'pepfar',
+                                                                    locations: location.to_s, rebuild: true.to_s)
+        cohort.find_report
+      end
+
+      def init_report
+        @report = initialize_report_structure
+      end
+
+      GENDER = %w[Male Female].freeze
+
+      def initialize_report_structure
+        pepfar_age_groups.each_with_object({}) do |age_group, report|
+          report[age_group] = GENDER.each_with_object({}) do |gender, age_group_report|
+            age_group_report[gender] = initialize_gender_metrics
+          end
+        end
+      end
+
+      def addittional_groups
+        @report['All'] = {}
+        %w[Male FP FNP FBf].each do |key|
+          @report['All'][key] = initialize_gender_metrics
+        end
+      end
+
+      def initialize_gender_metrics
+        {
+          died: [],
+          iit_less_3_months: [],
+          iit_3_to_5_months: [],
+          iit_6_plus_months: [],
+          transfer: [],
+          stopped: []
+        }
+      end
+
+      def flatten_the_report
+        result = []
+        @report.each do |age_group, age_group_report|
+          next if age_group == 'Unknown'
+
+          age_group_report.each_key do |gender|
+            next if gender == 'Unknown'
+
+            result << process_age_group_report(age_group, gender, age_group_report[gender])
+          end
+        end
+
+        new_group = pepfar_age_groups.map { |age_group| age_group }
+        new_group << 'All'
+        gender_scores = { 'Female' => 0, 'Male' => 1, 'FNP' => 3, 'FP' => 2, 'FBf' => 4 }
+        result_scores = result.sort_by do |item|
+          gender_score = gender_scores[item[:gender]] || 999
+          age_group_score = new_group.index(item[:age_group]) || 999
+          [gender_score, age_group_score]
+        end
+        # remove all unknown age groups
+        result_scores.reject { |item| item[:age_group].match?(/unknown/i) }
+      end
+
+      def process_age_group_report(age_group, gender, age_group_report)
+        {
+          age_group:,
+          gender:,
+          died: age_group_report[:died],
+          iit_less_3_months: age_group_report[:iit_less_3_months],
+          iit_3_to_5_months: age_group_report[:iit_3_to_5_months],
+          iit_6_plus_months: age_group_report[:iit_6_plus_months],
+          transfer: age_group_report[:transfer],
+          stopped: age_group_report[:stopped]
+        }
+      end
+
       def process_data
-        data = {}
         (process_tx_ml_clients || []).each do |pat|
           patient_id = pat['patient_id'].to_i
           outcome = pat['outcome']
-          gender = begin
-            pat['gender'].first.upcase
-          rescue StandardError
-            'Unknown'
-          end
+          gender = pat['gender'] || 'Unknown'
           age_group = pat['age_group']
+          maternal_status = pat['maternal_status']
+          next unless gender.in?(%w[Male Female])
 
-          if data[age_group].blank?
-            data[age_group] = {}
-            data[age_group][gender] = [[], [], [], [], [], []]
-          elsif data[age_group][gender].blank?
-            data[age_group][gender] = [[], [], [], [], [], []]
-          end
+          indicator = process_outcome(outcome, pat['months'])
+          aggregation(patient_id, gender, indicator, maternal_status, age_group)
+        rescue StandardError => e
+          Rails.logger.error "Error processing patient: #{patient_id} - #{age_group} - #{gender} - #{outcome} - #{e.message}"
+        end
+      end
 
-          case outcome
-          when 'Defaulted'
-            def_months = pat['months'].to_i
-            if def_months < 3
-              data[age_group][gender][1] << patient_id
-            elsif def_months <= 5
-              data[age_group][gender][2] << patient_id
-            elsif def_months > 5
-              data[age_group][gender][3] << patient_id
-            end
-          when 'Patient died'
-            data[age_group][gender][0] << patient_id
-          when /Stopped/i
-            data[age_group][gender][5] << patient_id
-          when 'Patient transferred out'
-            data[age_group][gender][4] << patient_id
+      def process_outcome(outcome, months)
+        case outcome
+        when 'Defaulted'
+          if months < 3
+            'iit_less_3_months'
+          elsif months <= 5
+            'iit_3_to_5_months'
+          elsif months > 5
+            'iit_6_plus_months'
           end
-        rescue StandardError => e
-          Rails.logger.error(e.message)
+        when 'Patient died'
+          'died'
+        when /Stopped/i
+          'stopped'
+        when 'Patient transferred out'
+          'transfer'
+        end
+      end
+
+      def aggregation(patient_id, gender, indicator, maternal_status, age_group)
+        @report[age_group][gender][indicator.to_sym] << patient_id
+        if gender == 'Male'
+          @report['All'][gender][indicator.to_sym] << patient_id
+          return
+        end
+
+        case maternal_status
+        when 'FP'
+          @report['All']['FP'][indicator.to_sym] << patient_id
+        when 'FBf'
+          @report['All']['FBf'][indicator.to_sym] << patient_id
+        else
+          @report['All']['FNP'][indicator.to_sym] << patient_id
         end
-        data
       end
 
       def tx_ml_clients
@@ -74,7 +158,12 @@ module MalawiHivProgramReports
           SELECT
             e.patient_id,
             e.birthdate,
-            e.gender,
+            CASE e.gender
+              WHEN 'M' THEN 'Male'
+              WHEN 'F' THEN 'Female'
+              ELSE 'Unknown'
+            END gender,
+            c.maternal_status,
             e.date_enrolled,
             e.earliest_start_date,
             o.outcome_date,
@@ -82,7 +171,8 @@ module MalawiHivProgramReports
             disaggregated_age_group(e.birthdate, DATE('#{end_date}')) age_group,
             o.cum_outcome outcome
           FROM cdr_temp_cohort_members #{current_partition} e
-          INNER JOIN cdr_temp_patient_outcomes #{current_partition} o ON e.patient_id = o.patient_id AND o.cum_outcome IN ('Defaulted', 'Patient died', 'Treatment stopped', 'Patient transferred out')
+          INNER JOIN cdr_temp_patient_outcomes #{current_partition} o ON e.patient_id = o.patient_id AND o.cum_outcome IN ('Defaulted', 'Patient died', 'Treatment Stopped', 'Patient transferred out')
+          LEFT JOIN cdr_temp_maternal_status #{current_partition} c ON c.patient_id = e.patient_id
           WHERE e.patient_id IN (SELECT patient_id FROM cdr_temp_patient_outcomes_start #{current_partition} WHERE cum_outcome = 'On antiretrovirals')
             AND DATE(e.earliest_start_date) < '#{start_date.to_date}'
           GROUP BY e.patient_id
@@ -94,7 +184,12 @@ module MalawiHivProgramReports
           SELECT
             e.patient_id,
             e.birthdate,
-            e.gender,
+            CASE e.gender
+              WHEN 'M' THEN 'Male'
+              WHEN 'F' THEN 'Female'
+              ELSE 'Unknown'
+            END gender,
+            c.maternal_status,
             e.date_enrolled,
             e.earliest_start_date,
             o.outcome_date,
@@ -102,7 +197,8 @@ module MalawiHivProgramReports
             disaggregated_age_group(e.birthdate, DATE('#{end_date}')) age_group,
             o.cum_outcome outcome
           FROM cdr_temp_cohort_members #{current_partition} e
-          INNER JOIN cdr_temp_patient_outcomes #{current_partition} o ON e.patient_id = o.patient_id AND o.cum_outcome IN ('Defaulted', 'Patient died', 'Treatment stopped', 'Patient transferred out')
+          INNER JOIN cdr_temp_patient_outcomes #{current_partition} o ON e.patient_id = o.patient_id AND o.cum_outcome IN ('Defaulted', 'Patient died', 'Treatment Stopped', 'Patient transferred out')
+          LEFT JOIN cdr_temp_maternal_status #{current_partition} c ON c.patient_id = e.patient_id
           WHERE e.earliest_start_date BETWEEN DATE('#{start_date}') AND DATE('#{end_date}')
           GROUP BY e.patient_id
         SQL

data/app/services/malawi_hiv_program_reports/pepfar/tx_new.rb

@@ -21,11 +21,7 @@ module MalawiHivProgramReports
       def find_report
         report = init_report
         addittional_groups report
-        if rebuild
-          cohort = MalawiHivProgramReports::Moh::CumulativeCohort.new(start_date: start_date.to_date, end_date: end_date.to_date, definition: 'pepfar',
-                                                                      locations: location.to_s, rebuild: true.to_s)
-          cohort.find_report
-        end
+        process_rebuild if rebuild
         process_data report
         flatten_the_report report
       rescue StandardError => e
@@ -37,17 +33,18 @@ module MalawiHivProgramReports
 
       private
 
-      GENDER = %w[M F].freeze
+      GENDER = %w[Male Female].freeze
+
+      def process_rebuild
+        cohort = MalawiHivProgramReports::Moh::CumulativeCohort.new(start_date: start_date.to_date, end_date: end_date.to_date, definition: 'pepfar',
+                                                                    locations: location.to_s, rebuild: true.to_s)
+        cohort.find_report
+      end
 
       def init_report
         pepfar_age_groups.each_with_object({}) do |age_group, report|
           report[age_group] = GENDER.each_with_object({}) do |gender, age_group_report|
-            age_group_report[gender] = {
-              cd4_less_than_200: [],
-              cd4_greater_than_equal_to_200: [],
-              cd4_unknown_or_not_done: [],
-              transfer_in: []
-            }
+            age_group_report[gender] = initialize_gender_metrics
           end
         end
       end
@@ -55,15 +52,19 @@ module MalawiHivProgramReports
       def addittional_groups(report)
         report['All'] = {}
         %w[Male FP FNP FBf].each do |key|
-          report['All'][key] = {
-            cd4_less_than_200: [],
-            cd4_greater_than_equal_to_200: [],
-            cd4_unknown_or_not_done: [],
-            transfer_in: []
-          }
+          report['All'][key] = initialize_gender_metrics
         end
       end
 
+      def initialize_gender_metrics
+        {
+          cd4_less_than_200: [],
+          cd4_greater_than_equal_to_200: [],
+          cd4_unknown_or_not_done: [],
+          transfer_in: []
+        }
+      end
+
       # rubocop:disable Metrics/AbcSize
       # rubocop:disable Metrics/MethodLength
       # rubocop:disable Metrics/PerceivedComplexity
@@ -72,7 +73,6 @@ module MalawiHivProgramReports
         fetch_data.each do |row|
           age_group = row['age_group']
           gender = row['gender']
-          date_enrolled = row['date_enrolled']
           next if age_group.blank?
           next if gender.blank?
           next unless GENDER.include?(gender)
@@ -83,32 +83,36 @@ module MalawiHivProgramReports
           patient_id = row['patient_id'].to_i
           earliest_start_date = row['earliest_start_date']
           indicator = new_patient.positive? ? cd4_count_group : 'transfer_in'
+          maternal_status = row['maternal_status']
 
-          if new_patient.positive? && earliest_start_date.to_date >= start_date.to_date
-            report[age_group.to_s][gender.to_s][indicator.to_sym] << patient_id
-          elsif new_patient.zero?
-            report[age_group.to_s][gender.to_s][indicator.to_sym] << patient_id
-          else
-            next
-          end
-          process_aggreggation_rows(report:, gender:, indicator:, start_date: date_enrolled,
-                                    patient_id:, maternal_status: row['maternal_status'], maternal_status_date: row['maternal_status_date'])
+          next if !(new_patient.positive? && earliest_start_date.to_date >= start_date.to_date) && !new_patient.zero?
+
+          process_aggreggation_rows(report:, gender:, indicator:, patient_id:, maternal_status:, age_group:)
         end
       end
 
-      def process_aggreggation_rows(report:, gender:, indicator:, start_date:, **kwargs)
+      def process_aggreggation_rows(report:, gender:, indicator:, **kwargs)
         maternal_status = kwargs[:maternal_status]
-        maternal_status_date = kwargs[:maternal_status_date]
+        age_group = kwargs[:age_group]
+        report[age_group][gender][indicator.to_sym] << kwargs[:patient_id]
 
-        if gender == 'M'
+        if gender == 'Male'
           report['All']['Male'][indicator.to_sym] << kwargs[:patient_id]
-        elsif maternal_status&.match?(/pregnant/i) && (maternal_status_date&.to_date&.<= start_date.to_date)
+          return
+        end
+
+        case maternal_status
+        when 'FP'
           report['All']['FP'][indicator.to_sym] << kwargs[:patient_id]
-        elsif maternal_status&.match?(/breast/i) && (maternal_status_date&.to_date&.<= start_date.to_date)
+        when 'FBf'
           report['All']['FBf'][indicator.to_sym] << kwargs[:patient_id]
         else
           report['All']['FNP'][indicator.to_sym] << kwargs[:patient_id]
         end
+      rescue StandardError => e
+        Rails.logger.error "Error processing patient: #{kwargs[:patient_id]} - #{gender} - #{indicator} - #{kwargs[:maternal_status]} - #{e.message}"
+        Rails.logger.error report
+        raise e
       end
       # rubocop:enable Metrics/PerceivedComplexity
       # rubocop:enable Metrics/CyclomaticComplexity
@@ -116,15 +120,8 @@ module MalawiHivProgramReports
       def process_age_group_report(age_group, gender, age_group_report)
         {
           age_group:,
-          gender: if gender == 'F'
-                    'Female'
-                  else
-                    (gender == 'M' ? 'Male' : gender)
-                  end,
-          cd4_less_than_200: age_group_report[:cd4_less_than_200],
-          cd4_greater_than_equal_to_200: age_group_report[:cd4_greater_than_equal_to_200],
-          cd4_unknown_or_not_done: age_group_report[:cd4_unknown_or_not_done],
-          transfer_in: age_group_report[:transfer_in]
+          gender:,
+          **age_group_report
         }
       end
       # rubocop:enable Metrics/AbcSize
@@ -133,6 +130,8 @@ module MalawiHivProgramReports
       def flatten_the_report(report)
         result = []
         report.each do |age_group, age_group_report|
+          next if age_group == 'Unknown'
+
           age_group_report.each_key do |gender|
             result << process_age_group_report(age_group, gender, age_group_report[gender])
           end
@@ -154,7 +153,11 @@ module MalawiHivProgramReports
         ActiveRecord::Base.connection.select_all <<~SQL
           SELECT
             e.patient_id,
-            e.gender,
+            CASE e.gender
+              WHEN 'M' THEN 'Male'
+              WHEN 'F' THEN 'Female'
+              ELSE 'Unknown'
+            END gender,
             disaggregated_age_group(e.birthdate, DATE('#{end_date}')) age_group,
             CASE
               WHEN o.value_numeric < 200 THEN 'cd4_less_than_200'
@@ -170,28 +173,12 @@ module MalawiHivProgramReports
             END new_patient,
             e.date_enrolled,
             e.earliest_start_date,
-            preg_or_breast.name AS maternal_status,
-            DATE(MIN(pregnant_or_breastfeeding.obs_datetime)) AS maternal_status_date
+            c.maternal_status
           FROM cdr_temp_cohort_members #{current_partition} e
-          LEFT JOIN (
-            SELECT max(o.obs_datetime) AS obs_datetime, o.person_id
-            FROM obs #{current_partition} o
-            INNER JOIN concept_name cn ON cn.concept_id = o.concept_id AND cn.name = 'CD4 count' AND cn.voided = 0
-            INNER JOIN patient_program #{current_partition} pp ON pp.patient_id = o.person_id
-                AND pp.program_id = #{program('HIV PROGRAM').id}
-                AND pp.voided = 0
-            INNER JOIN patient_state #{current_partition} ps ON ps.patient_program_id = pp.patient_program_id AND ps.voided = 0 AND ps.state = 7 AND ps.start_date <= DATE('#{end_date}')
-            WHERE o.concept_id = #{concept_name('CD4 count').concept_id} AND o.voided = 0
-            AND o.obs_datetime <= '#{end_date}' AND o.obs_datetime >= '#{start_date}'
-            GROUP BY o.person_id
-          ) current_cd4 ON current_cd4.person_id = e.patient_id
-          LEFT JOIN obs #{current_partition} o ON o.person_id = e.patient_id AND o.concept_id = #{concept_name('CD4 count').concept_id} AND o.voided = 0 AND o.obs_datetime = current_cd4.obs_datetime
-          LEFT JOIN obs #{current_partition} pregnant_or_breastfeeding ON pregnant_or_breastfeeding.person_id = e.patient_id
-            AND pregnant_or_breastfeeding.concept_id IN (SELECT concept_id FROM concept_name WHERE name IN ('Breast feeding?', 'Breast feeding', 'Breastfeeding', 'Is patient pregnant?', 'patient pregnant') AND voided = 0)
-            AND pregnant_or_breastfeeding.voided = 0
-            AND pregnant_or_breastfeeding.value_coded = #{concept_name('Yes').concept_id}
-          LEFT JOIN concept_name preg_or_breast ON preg_or_breast.concept_id = pregnant_or_breastfeeding.concept_id AND preg_or_breast.voided = 0
-          WHERE e.date_enrolled <= '#{end_date}' AND e.date_enrolled >= '#{start_date}'
+          LEFT JOIN obs #{current_partition} current_cd4 ON current_cd4.person_id = e.patient_id AND current_cd4.concept_id = #{concept_name('CD4 count').concept_id} AND current_cd4.voided = 0
+          LEFT JOIN obs #{current_partition} o ON o.person_id = current_cd4.person_id AND o.concept_id = #{concept_name('CD4 count').concept_id} AND o.voided = 0 AND o.obs_datetime > current_cd4.obs_datetime
+          LEFT JOIN cdr_temp_maternal_status #{current_partition} c ON c.patient_id = e.patient_id
+          WHERE e.date_enrolled <= '#{end_date}' AND e.date_enrolled >= '#{start_date}' AND e.gender IN ('M', 'F') AND o.obs_id IS NULL
           GROUP BY e.patient_id
         SQL
       end

data/app/services/malawi_hiv_program_reports/pepfar/tx_rtt.rb

@@ -7,7 +7,7 @@ module MalawiHivProgramReports
       include MalawiHivProgramReports::Utils::CommonSqlQueryUtils
       include MalawiHivProgramReports::Adapters::Moh::Custom
       include MalawiHivProgramReports::Utils::ModelUtils
-      attr_reader :start_date, :end_date, :location, :rebuild
+      attr_reader :start_date, :end_date, :location, :rebuild, :report
 
       def initialize(start_date:, end_date:, **kwargs)
         @start_date = ActiveRecord::Base.connection.quote(start_date.to_date.beginning_of_day.strftime('%Y-%m-%d %H:%M:%S'))
@@ -17,20 +17,16 @@ module MalawiHivProgramReports
       end
 
       def find_report
-        if rebuild
-          cohort = MalawiHivProgramReports::Moh::CumulativeCohort.new(start_date: start_date.to_date, end_date: end_date.to_date, definition: 'pepfar',
-                                                                      locations: location.to_s, rebuild: true.to_s)
-          cohort.find_report
-        end
+        process_rebuild
         process_report
+      rescue StandardError => e
+        Rails.logger.error "Error running TX_RTT Report: #{e}"
+        Rails.logger.error e.backtrace.join("\n")
+        raise e
       end
 
       def data
-        if rebuild
-          cohort = MalawiHivProgramReports::Moh::CumulativeCohort.new(start_date: start_date.to_date, end_date: end_date.to_date, definition: 'pepfar',
-                                                                      locations: location.to_s, rebuild: true.to_s)
-          cohort.find_report
-        end
+        process_rebuild
         process_report
       rescue StandardError => e
         Rails.logger.error "Error running TX_RTT Report: #{e}"
@@ -40,7 +36,15 @@ module MalawiHivProgramReports
 
       private
 
-      GENDER = %w[M F].freeze
+      GENDER = %w[Male Female].freeze
+
+      def process_rebuild
+        return unless rebuild
+
+        cohort = MalawiHivProgramReports::Moh::CumulativeCohort.new(start_date: start_date.to_date, end_date: end_date.to_date, definition: 'pepfar',
+                                                                    locations: location.to_s, rebuild: true.to_s)
+        cohort.find_report
+      end
 
       def init_report
         pepfar_age_groups.each_with_object({}) do |age_group, age_group_report|
@@ -50,44 +54,74 @@ module MalawiHivProgramReports
         end
       end
 
+      def addittional_groups
+        @report['All'] = {}
+        %w[Male FP FNP FBf].each do |key|
+          @report['All'][key] = indicators
+        end
+      end
+
       def process_report
-        report = init_report
-        process_data report
-        flatten_the_report report
+        @report = init_report
+        addittional_groups
+        process_data
+        flatten_the_report
       end
 
-      def process_data(report)
+      def process_data
         fetch_data.each do |row|
           age_group = row['age_group']
           gender = row['gender']
           months = row['months']
           patient_id = row['patient_id']
           cd4_cat = row['cd4_count_group']
+          maternal_status = row['maternal_status']
 
           next unless GENDER.include?(gender)
           next unless pepfar_age_groups.include?(age_group)
 
-          cat = report[age_group][gender]
-          process_months(cat, months, patient_id)
-          process_cd4(cat, months, patient_id, cd4_cat)
+          month_indicator = process_months(months)
+          cd4_indicator = process_cd4(months, cd4_cat)
+          process_aggreggation_rows(age_group:, gender:, month_indicator:, cd4_indicator:, maternal_status:,
+                                    patient_id:)
         end
       end
 
-      def process_months(report, months, patient_id)
-        return report[:returned_less_than_3_months] << patient_id if months.blank?
-        return report[:returned_less_than_3_months] << patient_id if months < 3
-        return report[:returned_greater_than_3_months_and_less_than_6_months] << patient_id if months >= 3 && months < 6
+      def process_aggreggation_rows(age_group:, gender:, month_indicator:, cd4_indicator:, **kwargs)
+        @report[age_group][gender][month_indicator] << kwargs[:patient_id] unless month_indicator&.blank?
+        @report[age_group][gender][cd4_indicator] << kwargs[:patient_id] unless cd4_indicator&.blank?
 
-        report[:returned_greater_than_or_equal_to_6_months] << patient_id if months >= 6
-      end
-
-      def process_cd4(report, months, patient_id, cd4_cat)
-        if cd4_cat == 'unknown_cd4_count' && (months.blank? || months <= 2)
-          report[:not_eligible_for_cd4] << patient_id
+        if gender == 'Male'
+          @report['All']['Male'][month_indicator] << kwargs[:patient_id] unless month_indicator&.blank?
+          @report['All']['Male'][cd4_indicator] << kwargs[:patient_id] unless cd4_indicator&.blank?
           return
         end
 
-        report[cd4_cat.to_sym] << patient_id
+        case kwargs[:maternal_status]
+        when 'FP'
+          @report['All']['FP'][month_indicator] << kwargs[:patient_id] unless month_indicator&.blank?
+          @report['All']['FP'][month_indicator] << kwargs[:patient_id] unless cd4_indicator&.blank?
+        when 'FBf'
+          @report['All']['FBf'][month_indicator] << kwargs[:patient_id] unless month_indicator&.blank?
+          @report['All']['FBf'][cd4_indicator] << kwargs[:patient_id] unless cd4_indicator&.blank?
+        else
+          @report['All']['FNP'][month_indicator] << kwargs[:patient_id] unless month_indicator&.blank?
+          @report['All']['FNP'][cd4_indicator] << kwargs[:patient_id] unless cd4_indicator&.blank?
+        end
+      end
+
+      def process_months(months)
+        return :returned_less_than_3_months if months.blank?
+        return :returned_less_than_3_months if months < 3
+        return :returned_greater_than_3_months_and_less_than_6_months if months >= 3 && months < 6
+
+        :returned_greater_than_or_equal_to_6_months if months >= 6
+      end
+
+      def process_cd4(months, cd4_cat)
+        cd4_cat == 'unknown_cd4_count' if months.blank? || months <= 2
+
+        cd4_cat.to_sym
       end
 
       def indicators
@@ -106,29 +140,32 @@ module MalawiHivProgramReports
         {
           age_group:,
           gender:,
-          cd4_less_than_200: age_group_report[:cd4_less_than_200],
-          cd4_greater_than_or_equal_to_200: age_group_report[:cd4_greater_than_or_equal_to_200],
-          unknown_cd4_count: age_group_report[:unknown_cd4_count],
-          not_eligible_for_cd4: age_group_report[:not_eligible_for_cd4],
-          returned_less_than_3_months: age_group_report[:returned_less_than_3_months],
-          returned_greater_than_3_months_and_less_than_6_months: age_group_report[:returned_greater_than_3_months_and_less_than_6_months],
-          returned_greater_than_or_equal_to_6_months: age_group_report[:returned_greater_than_or_equal_to_6_months]
+          **age_group_report
         }
       end
 
-      def flatten_the_report(report)
+      def flatten_the_report
         result = []
-        report.each do |age_group, age_group_report|
-          result << process_age_group_report(age_group, 'M', age_group_report['M'])
-          result << process_age_group_report(age_group, 'F', age_group_report['F'])
+        @report.each do |age_group, age_group_report|
+          next if age_group == 'Unknown'
+
+          age_group_report.each_key do |gender|
+            next if gender == 'Unknown'
+
+            result << process_age_group_report(age_group, gender, age_group_report[gender])
+          end
         end
-        sorted_results = result.sort_by do |item|
-          gender_score = item[:gender] == 'Female' ? 0 : 1
-          age_group_score = pepfar_age_groups.index(item[:age_group])
+
+        new_group = pepfar_age_groups.map { |age_group| age_group }
+        new_group << 'All'
+        gender_scores = { 'Female' => 0, 'Male' => 1, 'FNP' => 3, 'FP' => 2, 'FBf' => 4 }
+        result_scores = result.sort_by do |item|
+          gender_score = gender_scores[item[:gender]] || 999
+          age_group_score = new_group.index(item[:age_group]) || 999
           [gender_score, age_group_score]
         end
-        # sort by gender, start all females and push all males to the end
-        sorted_results.sort_by { |h| [h[:gender] == 'F' ? 0 : 1] }
+        # remove all unknown age groups
+        result_scores.reject { |item| item[:age_group].match?(/unknown/i) }
       end
 
       def fetch_data
@@ -136,7 +173,12 @@ module MalawiHivProgramReports
           SELECT
             e.patient_id,
             disaggregated_age_group(e.birthdate, #{end_date}) AS age_group,
-            e.gender,
+            CASE e.gender
+              WHEN 'M' THEN 'Male'
+              WHEN 'F' THEN 'Female'
+              ELSE 'Unknown'
+            END gender,
+            ms.maternal_status,
             s.cum_outcome initial_outcome,
             o.cum_outcome final_outcome,
             TIMESTAMPDIFF(MONTH, COALESCE(s.outcome_date, c.outcome_date), ord.min_order_date) months,
@@ -153,7 +195,9 @@ module MalawiHivProgramReports
           INNER JOIN cdr_temp_patient_outcomes_start #{current_partition} s ON s.patient_id = e.patient_id AND s.cum_outcome IN ('Defaulted', 'Treatment stopped')
           INNER JOIN cdr_temp_current_state_start #{current_partition} c ON c.patient_id = e.patient_id
           INNER JOIN cdr_temp_max_drug_orders #{current_partition} ord ON ord.patient_id = e.patient_id
+          LEFT JOIN cdr_temp_maternal_status #{current_partition} ms ON ms.patient_id = e.patient_id
           LEFT JOIN obs #{current_partition} cd4_result ON cd4_result.person_id = e.patient_id AND cd4_result.concept_id = #{concept_name('CD4 count').concept_id} AND cd4_result.voided = 0
+          WHERE e.gender IN ('M', 'F')
           GROUP BY e.patient_id
         SQL
       end