malawi_hiv_program_reports 1.0.21 → 1.0.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9694980e1a8ca6e260fc5e92e61931383205f36a696fbdc9764514efca2291f2
-  data.tar.gz: 3b1c35e90152fca9414bd1d45c9d9e4b953c0278229a0e57f7aaeb249f776fb7
+  metadata.gz: 13a308e891673d2c03b82b355cbd1cb30acf9562bd4710e90b975563a7b9a5d0
+  data.tar.gz: 5a347a6889a9bf4cedd109e83df9146c21265b38e4aab1999ca1db3dca5f7fd6
 SHA512:
-  metadata.gz: 1f360402cf0528ca40b06ed4d72eca644c5d6852685c4fd7aa36a0cee3974608377b501558742f6d4fd7296f4ef7997c39b713072654c936e6d5852e4d49c164
-  data.tar.gz: 5e594587de91c93eb4dc3f6aaf4923aedd5d56a7d41a2bfbd62c39b88f6a1ede319a2a33be40cecfc4b74ac5f4af9708882592cdaf00a0239751ae369f9c984f
+  metadata.gz: f918d797106fafc2c0ea27b5f529dbee2403583be90428e679ec27b5d2bd4bc97a541c5ee369cf7122ec9b52d63be1bb01331833e40a7c7e8ebbc950d612d319
+  data.tar.gz: 882a731d6f7f538e3d4cbd81fe9052ac54b15dd5b304b49d27abb097876de56daf9d4e6fcbbe5310fd69fa9003fdb3e01834071e7732afcf292ea56f895e11da

data/app/services/malawi_hiv_program_reports/moh/cumulative_cohort.rb

@@ -27,7 +27,7 @@ module MalawiHivProgramReports
         process_thread
         end_time = Time.now
         time_in_minutes = ((end_time - start_time) / 60).round(2)
-        Rails.logger.info("Cumulative Cohort report took #{time_in_minutes} minutes to generate")
+        Rails.logger.info("Cumulative Cohort report took #{time_in_minutes} minutes to generate for these locations: #{locations}")
         { cohort_time: time_in_minutes }
       end
 
@@ -47,7 +47,7 @@ module MalawiHivProgramReports
         queue = Queue.new
         locations.each { |loc| queue << loc }
 
-        threads = Array.new(10) do
+        threads = Array.new(40) do
           Thread.new do
             until queue.empty?
               loc = begin
@@ -59,6 +59,9 @@ module MalawiHivProgramReports
 
               ActiveRecord::Base.connection_pool.with_connection do
                 process_data loc
+              rescue StandardError => e
+                Rails.logger.error("Error processing location #{loc}: #{e.message}")
+                save_incomplete_site(location: loc, time_taken: 0)
               end
             end
           end
@@ -104,6 +107,13 @@ module MalawiHivProgramReports
         SQL
       end
 
+      def save_incomplete_site(location:, time_taken:)
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO cdr_temp_cohort_status
+          VALUES (#{location}, DATE(#{start_date}), DATE(#{end_date}), 'incomplete', #{time_taken})
+        SQL
+      end
+
       def create_cdr_temp_cohort_status
         ActiveRecord::Base.connection.execute <<~SQL
           CREATE TABLE IF NOT EXISTS cdr_temp_cohort_status (

data/app/services/malawi_hiv_program_reports/moh/cumulative_outcome.rb

@@ -24,14 +24,13 @@ module MalawiHivProgramReports
 
       private
 
-      # The main idea here is to come up with cumulative outcomes for patients in cdr_temp_cohort_members
+      # The main idea here is to come up with cumulative outcomes for patients in temp_earliest_start_date
       # 1. load_patients_who_died
       # 2. load_patients_who_stopped_treatment
-      # 3. load_patients_on_pre_art
-      # 4. load_patients_without_state
-      # 5. load_patients_without_drug_orders
-      # 6. load_patients_on_treatment
-      # 7. load_defaulters
+      # 3. load_patients_without_drug_orders
+      # 4. load_patients_on_treatment
+      # 5. load_without_clinical_contact
+      # 6. load_defaulters
 
       def program_states(*names)
         ::ProgramWorkflowState.joins(:program_workflow)
@@ -47,22 +46,27 @@ module MalawiHivProgramReports
       # ===================================
       # rubocop:disable Metrics/MethodLength
       def process_data
-        load_max_drug_orders
-        load_min_auto_expire_date
-        load_max_patient_state
-        load_max_appointment_date
+        denormalize
         # HIC SUNT DRACONIS: The order of the operations below matters,
         # do not change it unless you know what you are doing!!!
         load_patients_who_died
         load_patients_who_stopped_treatment
-        load_patients_on_pre_art
-        load_patients_without_state
         load_patients_without_drug_orders
         load_patients_on_treatment
         load_without_clinical_contact
         load_defaulters
       end
 
+      def denormalize
+        load_max_drug_orders
+        load_patient_current_medication
+        update_patient_current_medication
+        load_min_auto_expire_date
+        load_max_patient_state
+        load_patient_current_state
+        update_patient_current_state
+      end
+
       def load_max_drug_orders
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_max_drug_orders PARTITION (p#{location})
@@ -80,34 +84,67 @@ module MalawiHivProgramReports
         SQL
       end
 
-      def load_min_auto_expire_date
+      def load_patient_current_medication
         ActiveRecord::Base.connection.execute <<~SQL
-          INSERT INTO cdr_temp_min_auto_expire_date PARTITION (p#{location})
-          SELECT patient_id, o.site_id, MIN(auto_expire_date) AS auto_expire_date
-          FROM orders o
-          INNER JOIN cdr_temp_max_drug_orders USING (patient_id, site_id, start_date)
-          INNER JOIN drug_order ON drug_order.order_id = o.order_id
-            AND drug_order.site_id = o.site_id AND drug_order.quantity > 0
-            AND drug_order.drug_inventory_id IN (#{arv_drug})
-          WHERE o.order_type_id = 1
-            AND o.voided = 0 AND o.site_id = #{location}
-          GROUP BY patient_id, o.site_id
-          HAVING auto_expire_date IS NOT NULL
-          ON DUPLICATE KEY UPDATE auto_expire_date = VALUES(auto_expire_date)
+          INSERT INTO cdr_temp_current_medication PARTITION (p#{location})
+          SELECT mdo.patient_id, mdo.site_id, d.concept_id, do.drug_inventory_id drug_id,
+            CASE
+              WHEN do.equivalent_daily_dose IS NULL THEN 1
+              WHEN do.equivalent_daily_dose = 0 THEN 1
+              WHEN do.equivalent_daily_dose REGEXP '^[0-9]+(.[0-9]+)?$' THEN do.equivalent_daily_dose
+              ELSE 1
+            END daily_dose,
+            SUM(do.quantity) quantity,
+            DATE(mdo.start_date) start_date, null, null, null, null
+          FROM cdr_temp_max_drug_orders mdo
+          INNER JOIN orders o ON o.patient_id = mdo.patient_id AND o.site_id = mdo.site_id AND o.order_type_id = 1 AND DATE(o.start_date) = DATE(mdo.start_date) AND o.voided = 0
+          INNER JOIN drug_order do ON do.order_id = o.order_id AND do.site_id = o.site_id AND do.quantity > 0 AND do.drug_inventory_id IN (#{arv_drug})
+          INNER JOIN drug d ON d.drug_id = do.drug_inventory_id
+          WHERE mdo.site_id = #{location}
+          GROUP BY mdo.patient_id, do.drug_inventory_id HAVING quantity < 6000
+          ON DUPLICATE KEY UPDATE concept_id = VALUES(concept_id), daily_dose = VALUES(daily_dose), quantity=VALUES(quantity), start_date = VALUES(start_date), pill_count = VALUES(pill_count), expiry_date = VALUES(expiry_date), pepfar_defaulter_date = VALUES(pepfar_defaulter_date), moh_defaulter_date = VALUES(moh_defaulter_date)
         SQL
       end
 
-      def load_max_appointment_date
+      def update_patient_current_medication
         ActiveRecord::Base.connection.execute <<~SQL
-          INSERT INTO cdr_temp_max_patient_appointment PARTITION (p#{location})
-          SELECT o.person_id, o.site_id, DATE(MAX(o.value_datetime)) appointment_date
-          FROM obs o
-          INNER JOIN encounter e ON e.encounter_id = o.encounter_id AND e.site_id = o.site_id AND e.voided = 0
-            AND e.program_id = 1 AND e.encounter_datetime < DATE(#{end_date}) + INTERVAL 1 DAY
-          WHERE o.concept_id = 5096 AND o.voided = 0 AND o.obs_datetime < DATE(#{end_date}) + INTERVAL 1 DAY AND o.site_id = #{location}
-          GROUP BY o.person_id, o.site_id
-          HAVING appointment_date IS NOT NULL
-          ON DUPLICATE KEY UPDATE appointment_date = VALUES(appointment_date)
+          INSERT INTO cdr_temp_current_medication PARTITION (p#{location})
+          SELECT cm.patient_id, cm.site_id, cm.concept_id, cm.drug_id, cm.daily_dose, cm.quantity, cm.start_date,
+          COALESCE(first_ob.quantity, 0) + COALESCE(SUM(second_ob.value_numeric),0) + COALESCE(SUM(third_ob.value_numeric),0) AS pill_count,
+          DATE_ADD(DATE_SUB(cm.start_date, INTERVAL 1 DAY), INTERVAL (cm.quantity + COALESCE(first_ob.quantity, 0) + COALESCE(SUM(second_ob.value_numeric),0) + COALESCE(SUM(third_ob.value_numeric),0)) / cm.daily_dose DAY),
+          DATE_ADD(DATE_ADD(DATE_SUB(cm.start_date, INTERVAL 1 DAY), INTERVAL (cm.quantity + COALESCE(first_ob.quantity, 0) + COALESCE(SUM(second_ob.value_numeric),0) + COALESCE(SUM(third_ob.value_numeric),0)) / cm.daily_dose DAY), INTERVAL 30 DAY),
+          DATE_ADD(DATE_ADD(DATE_SUB(cm.start_date, INTERVAL 1 DAY), INTERVAL (cm.quantity + COALESCE(first_ob.quantity, 0) + COALESCE(SUM(second_ob.value_numeric),0) + COALESCE(SUM(third_ob.value_numeric),0)) / cm.daily_dose DAY), INTERVAL 60 DAY)
+          FROM cdr_temp_current_medication cm
+          LEFT JOIN (
+          SELECT ob.person_id,ob.site_id, cm.drug_id,
+            SUM(ob.value_numeric) + SUM(CASE
+              WHEN ob.value_text is null then 0
+              WHEN ob.value_text REGEXP '^[0-9]+(.[0-9]+)?$' then ob.value_text
+              ELSE 0
+            END) quantity
+          FROM obs ob
+          INNER JOIN cdr_temp_current_medication cm ON cm.patient_id = ob.person_id AND cm.site_id = ob.site_id AND cm.start_date = DATE(ob.obs_datetime)
+          INNER JOIN orders o ON o.order_id = ob.order_id AND o.site_id = ob.site_id AND o.voided = 0
+          INNER JOIN drug_order do ON do.order_id = o.order_id AND do.site_id = o.site_id AND do.drug_inventory_id = cm.drug_id
+          WHERE ob.concept_id = 2540 AND ob.voided = 0 AND ob.site_id = #{location}
+          GROUP BY ob.person_id, cm.drug_id
+          ) first_ob ON first_ob.person_id = cm.patient_id AND first_ob.site_id = cm.site_id AND first_ob.drug_id = cm.drug_id
+          LEFT JOIN obs second_ob ON second_ob.person_id = cm.patient_id AND second_ob.site_id = cm.site_id  AND second_ob.concept_id = cm.concept_id AND DATE(second_ob.obs_datetime) = cm.start_date AND second_ob.voided = 0
+          LEFT JOIN obs third_ob ON third_ob.person_id = cm.patient_id AND third_ob.site_id = cm.site_id AND third_ob.concept_id = 2540 AND third_ob.value_drug = cm.drug_id AND third_ob.voided = 0 AND DATE(third_ob.obs_datetime) = cm.start_date
+          WHERE cm.site_id = #{location}
+          GROUP BY cm.patient_id, cm.drug_id
+          ON DUPLICATE KEY UPDATE pill_count = VALUES(pill_count), expiry_date = VALUES(expiry_date), pepfar_defaulter_date = VALUES(pepfar_defaulter_date), moh_defaulter_date = VALUES(moh_defaulter_date);
+        SQL
+      end
+
+      def load_min_auto_expire_date
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO cdr_temp_min_auto_expire_date PARTITION (p#{location})
+          SELECT cm.patient_id, cm.site_id, MIN(cm.start_date), MIN(cm.expiry_date), MIN(cm.pepfar_defaulter_date), MIN(cm.moh_defaulter_date)
+          FROM cdr_temp_current_medication cm
+          WHERE cm.site_id = #{location}
+          GROUP BY cm.patient_id
+          ON DUPLICATE KEY UPDATE start_date = VALUES(start_date), auto_expire_date = VALUES(auto_expire_date), pepfar_defaulter_date = VALUES(pepfar_defaulter_date), moh_defaulter_date = VALUES(moh_defaulter_date)
         SQL
       end
 
@@ -118,13 +155,43 @@ module MalawiHivProgramReports
           FROM patient_state ps
           INNER JOIN patient_program pp ON pp.patient_program_id = ps.patient_program_id AND pp.site_id = ps.site_id AND pp.program_id = 1 AND pp.voided = 0
           WHERE ps.start_date < DATE(#{end_date}) + INTERVAL 1 DAY
-            AND ps.voided = 0 AND ps.site_id = #{location}
+            AND ps.voided = 0 AND ps.site_id = #{location} AND pp.patient_id IN (SELECT patient_id FROM temp_earliest_start_date WHERE site_id = #{location})
           GROUP BY pp.patient_id, pp.site_id
           HAVING start_date IS NOT NULL
           ON DUPLICATE KEY UPDATE start_date = VALUES(start_date)
         SQL
       end
 
+      def load_patient_current_state
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO cdr_temp_current_state PARTITION (p#{location})
+          SELECT mps.patient_id, mps.site_id, cn.name AS cum_outcome, ps.start_date as outcome_date, ps.state, count(DISTINCT(ps.state)) outcomes, MAX(ps.patient_state_id) patient_state_id
+          FROM cdr_temp_max_patient_state mps
+          INNER JOIN patient_program pp ON pp.patient_id = mps.patient_id AND pp.site_id = mps.site_id AND pp.program_id = 1 AND pp.voided = 0
+          INNER JOIN patient_state ps ON ps.patient_program_id = pp.patient_program_id AND ps.site_id = pp.site_id  AND ps.start_date = mps.start_date AND ps.voided = 0
+            AND (ps.end_date IS NULL OR ps.end_date > DATE(#{end_date}))
+          INNER JOIN program_workflow_state pws ON pws.program_workflow_state_id = ps.state AND pws.retired = 0
+          INNER JOIN concept_name cn ON cn.concept_id = pws.concept_id AND cn.concept_name_type = 'FULLY_SPECIFIED' AND cn.voided = 0
+          LEFT JOIN patient_state ps2 ON ps.patient_program_id = ps2.patient_program_id AND ps2.site_id = ps.site_id AND ps.start_date = ps2.start_date AND ps.date_created < ps2.date_created AND ps2.voided = 0
+          WHERE ps2.patient_program_id IS NULL AND ps.voided = 0 AND mps.site_id = #{location}
+          GROUP BY mps.patient_id
+          ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), state = VALUES(state), outcomes = VALUES(outcomes)
+        SQL
+      end
+
+      def update_patient_current_state
+        ActiveRecord::Base.connection.execute <<~SQL
+          INSERT INTO cdr_temp_current_state
+          SELECT cs.patient_id, cs.site_id, cn.name as cum_outcome, ps.start_date as outcome_date, ps.state, 1, cs.patient_state_id
+          FROM patient_state ps
+          INNER JOIN cdr_temp_current_state cs ON cs.patient_state_id = ps.patient_state_id AND cs.site_id = ps.site_id
+          INNER JOIN program_workflow_state pws ON pws.program_workflow_state_id = ps.state AND pws.retired = 0
+          INNER JOIN concept_name cn ON cn.concept_id = pws.concept_id AND cn.concept_name_type = 'FULLY_SPECIFIED' AND cn.voided = 0
+          WHERE ps.voided = 0 AND cs.outcomes > 1 AND cs.site_id = #{location}
+          ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), state = VALUES(state), outcomes = VALUES(outcomes), patient_state_id = VALUES(patient_state_id)
+        SQL
+      end
+
       # Loads all patiens with an outcome of died as of given date
       # into the temp_patient_outcomes table.
       def load_patients_who_died
@@ -160,98 +227,20 @@ module MalawiHivProgramReports
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_patient_outcomes PARTITION (p#{location})
           SELECT patients.patient_id,
-                 (
-                   SELECT name FROM concept_name
-                   WHERE concept_id = (
-                   SELECT concept_id FROM program_workflow_state
-                   WHERE program_workflow_state_id = patient_state.state
-                   LIMIT 1
-                   )
-                 ) AS cum_outcome,
-                 patient_state.start_date, patients.site_id, 2
-          FROM cdr_temp_cohort_members AS patients
-          INNER JOIN patient_program
-            ON patient_program.patient_id = patients.patient_id
-            AND patient_program.site_id = patients.site_id
-            AND patient_program.program_id = 1
-            AND patient_program.voided = 0
-          INNER JOIN patient_state
-            ON patient_state.patient_program_id = patient_program.patient_program_id
-            AND patient_state.site_id = patient_program.site_id
-            AND patient_state.state IN (#{program_states('Patient transferred out', 'Treatment stopped').to_sql})
-            AND patient_state.start_date < DATE(#{end_date}) + INTERVAL 1 DAY
-            AND (patient_state.end_date >= #{end_date} OR patient_state.end_date IS NULL)
-            AND patient_state.voided = 0
-          INNER JOIN cdr_temp_max_patient_state AS max_patient_state
-            ON max_patient_state.patient_id = patient_program.patient_id
-            AND max_patient_state.site_id = patient_state.site_id
-            AND max_patient_state.start_date = patient_state.start_date
-          WHERE patients.date_enrolled <= #{end_date} AND patients.site_id = #{location}
-            AND (patients.patient_id, patients.site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step = 1)
-          GROUP BY patients.patient_id, patients.site_id
-          ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), step = VALUES(step)
-        SQL
-      end
-
-      # Load all patients on Pre-ART.
-      def load_patients_on_pre_art
-        ActiveRecord::Base.connection.execute <<~SQL
-          INSERT INTO cdr_temp_patient_outcomes PARTITION (p#{location})
-          SELECT patients.patient_id,
-                CASE
-                  WHEN #{current_defaulter_function('patients.patient_id', 'patients.site_id')} = 1 THEN 'Defaulted'
-                  ELSE 'Pre-ART (Continue)'
-                END AS cum_outcome,
-                patient_state.start_date, patients.site_id, 3
-          FROM cdr_temp_cohort_members AS patients
-          INNER JOIN patient_program
-            ON patient_program.patient_id = patients.patient_id
-            AND patient_program.site_id = patients.site_id
-            AND patient_program.program_id = 1
-            AND patient_program.voided = 0
-          INNER JOIN patient_state
-            ON patient_state.patient_program_id = patient_program.patient_program_id
-            AND patient_state.site_id = patient_program.site_id
-            AND patient_state.state = (#{program_states('Pre-ART (Continue)').limit(1).to_sql})
-            AND patient_state.start_date < DATE(#{end_date}) + INTERVAL 1 DAY
-            AND (patient_state.end_date >= #{end_date} OR patient_state.end_date IS NULL)
-            AND patient_state.voided = 0
-          INNER JOIN cdr_temp_max_patient_state AS max_patient_state
-            ON max_patient_state.patient_id = patient_program.patient_id
-            AND max_patient_state.site_id = patient_state.site_id
-            AND max_patient_state.start_date = patient_state.start_date
-          WHERE patients.date_enrolled <= #{end_date} AND patients.site_id = #{location}
-            AND (patients.patient_id, patients.site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2))
-          GROUP BY patients.patient_id, patients.site_id
-          ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), step = VALUES(step)
-        SQL
-      end
-
-      # Load all patients without a state
-      def load_patients_without_state
-        ActiveRecord::Base.connection.execute <<~SQL
-          INSERT INTO cdr_temp_patient_outcomes PARTITION (p#{location})
-          SELECT patients.patient_id,
-                 CASE
-                    WHEN #{current_defaulter_function('patients.patient_id', 'patients.site_id')} = 1 THEN 'Defaulted'
-                    ELSE 'Unknown'
-                  END AS cum_outcome,
-                 NULL, patients.site_id, 4
-          FROM cdr_temp_cohort_members AS patients
-          INNER JOIN patient_program
-            ON patient_program.patient_id = patients.patient_id
-            AND patient_program.site_id = patients.site_id
-            AND patient_program.program_id = 1
-            AND patient_program.voided = 0
-          WHERE patients.date_enrolled <= #{end_date} AND patients.site_id = #{location}
-            AND (patient_program.patient_program_id, patient_program.site_id) NOT IN (
-              SELECT patient_program_id, site_id
-              FROM patient_state
-              WHERE start_date < DATE(#{end_date}) + INTERVAL 1 DAY AND voided = 0
-            )
-            AND (patients.patient_id, patients.site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2, 3))
-          GROUP BY patients.patient_id, patients.site_id
-          HAVING cum_outcome = 'Defaulted'
+            patients.cum_outcome,
+            patients.outcome_date, patients.site_id, 2
+          FROM cdr_temp_current_state AS patients
+          WHERE (patients.patient_id) NOT IN (SELECT patient_id FROM cdr_temp_patient_outcomes PARTITION (p#{location})  WHERE step = 1)
+          AND patients.outcomes = 1
+          AND patients.site_id = #{location}
+          AND patients.state IN (
+            SELECT pws.program_workflow_state_id state
+            FROM program_workflow pw
+            INNER JOIN program_workflow_state pws ON pws.program_workflow_id = pw.program_workflow_id AND pws.retired = 0
+            WHERE pw.program_id = 1 AND pw.retired = 0 AND pws.terminal = 1
+            AND pws.program_workflow_state_id IN (2, 3, 6) -- Transferred out,Patient Died, Treatment stopped
+          )
+          GROUP BY patients.patient_id
           ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), step = VALUES(step)
         SQL
       end
@@ -261,9 +250,7 @@ module MalawiHivProgramReports
       def load_patients_without_drug_orders
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_patient_outcomes PARTITION (p#{location})
-          SELECT patients.patient_id,
-                 'Unknown',
-                 NULL, patients.site_id, 5
+          SELECT patients.patient_id, 'Unknown', NULL, patients.site_id, 3
           FROM cdr_temp_cohort_members AS patients
           WHERE date_enrolled <= #{end_date} AND site_id = #{location}
             AND (patient_id, site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2, 3, 4))
@@ -276,37 +263,12 @@ module MalawiHivProgramReports
       def load_patients_on_treatment
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_patient_outcomes PARTITION (p#{location})
-          SELECT patients.patient_id, 'On antiretrovirals', patient_state.start_date, patients.site_id, 6
-          FROM cdr_temp_cohort_members AS patients
-          INNER JOIN patient_program
-            ON patient_program.patient_id = patients.patient_id
-            AND patient_program.site_id = patients.site_id
-            AND patient_program.program_id = 1
-            AND patient_program.voided = 0
-          /* Get patients' `on ARV` states that are before given date */
-          INNER JOIN patient_state
-            ON patient_state.patient_program_id = patient_program.patient_program_id
-            AND patient_state.site_id = patient_program.site_id
-            AND patient_state.state = 7 -- ON ART
-            AND patient_state.start_date < DATE(#{end_date}) + INTERVAL 1 DAY
-            AND (patient_state.end_date >= #{end_date} OR patient_state.end_date IS NULL)
-            AND patient_state.voided = 0
-          /* Select only the most recent state out of those retrieved above */
-          INNER JOIN cdr_temp_max_patient_state AS max_patient_state
-            ON max_patient_state.patient_id = patient_program.patient_id
-            AND max_patient_state.site_id = patient_state.site_id
-            AND max_patient_state.start_date = patient_state.start_date
-          /* HACK: Ensure that the states captured above do correspond have corresponding
-                   ARV dispensations. In other words filter out any `on ARVs` states whose
-                   dispensation's may have been voided or states that were created manually
-                   without any drugs being dispensed.  */
-          INNER JOIN cdr_temp_min_auto_expire_date AS first_order_to_expire
-            ON first_order_to_expire.patient_id = patient_program.patient_id
-            AND first_order_to_expire.site_id = patient_program.site_id
-            AND (first_order_to_expire.auto_expire_date >= #{end_date} OR TIMESTAMPDIFF(DAY,first_order_to_expire.auto_expire_date, #{end_date}) <= #{@definition == 'pepfar' ? 28 : 56})
-          WHERE patients.date_enrolled <= #{end_date} AND patients.site_id = #{location}
-           AND (patients.patient_id, patients.site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2, 3, 4, 5))
-          GROUP BY patients.patient_id, patients.site_id
+          SELECT patients.patient_id, 'On antiretrovirals', COALESCE(cs.outcome_date, patients.start_date), patients.site_id, 4
+          FROM cdr_temp_min_auto_expire_date AS patients
+          LEFT JOIN cdr_temp_current_state AS cs ON cs.patient_id = patients.patient_id AND cs.site_id = patients.site_id
+          WHERE patients.#{@definition == 'pepfar' ? 'pepfar_defaulter_date' : 'moh_defaulter_date'} > #{end_date}
+          AND (patients.patient_id) NOT IN (SELECT patient_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2, 3))
+          AND patients.site_id = #{location}
           ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), step = VALUES(step)
         SQL
       end
@@ -314,38 +276,12 @@ module MalawiHivProgramReports
       def load_without_clinical_contact
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_patient_outcomes PARTITION (p#{location})
-          SELECT patients.patient_id, 'Defaulted', null, patients.site_id, 7
-          FROM cdr_temp_cohort_members AS patients
-          INNER JOIN patient_program
-            ON patient_program.patient_id = patients.patient_id
-            AND patient_program.site_id = patients.site_id
-            AND patient_program.program_id = 1
-            AND patient_program.voided = 0
-          /* Get patients' `on ARV` states that are before given date */
-          INNER JOIN patient_state
-            ON patient_state.patient_program_id = patient_program.patient_program_id
-            AND patient_state.site_id = patient_program.site_id
-            AND patient_state.state = 7 -- On ART
-            AND patient_state.start_date < DATE(#{end_date}) + INTERVAL 1 DAY
-            AND (patient_state.end_date >= #{end_date} OR patient_state.end_date IS NULL)
-            AND patient_state.voided = 0
-          /* Select only the most recent state out of those retrieved above */
-          INNER JOIN cdr_temp_max_patient_state AS max_patient_state
-            ON max_patient_state.patient_id = patient_program.patient_id
-            AND max_patient_state.site_id = patient_state.site_id
-            AND max_patient_state.start_date = patient_state.start_date
-          INNER JOIN cdr_temp_max_patient_appointment app ON app.patient_id = patients.patient_id
-          AND app.site_id = patients.site_id AND app.appointment_date < #{end_date}
-          INNER JOIN cdr_temp_min_auto_expire_date AS first_order_to_expire
-            ON first_order_to_expire.patient_id = patient_program.patient_id
-            AND first_order_to_expire.site_id = patient_program.site_id
-            AND TIMESTAMPDIFF(DAY,app.appointment_date, first_order_to_expire.auto_expire_date) >= 0
-            AND TIMESTAMPDIFF(DAY,app.appointment_date, first_order_to_expire.auto_expire_date) <= 5
-            AND first_order_to_expire.auto_expire_date < #{end_date}
-            AND TIMESTAMPDIFF(DAY,first_order_to_expire.auto_expire_date, #{end_date}) >= 365
-          WHERE patients.date_enrolled <= #{end_date} AND patients.site_id = #{location}
-          AND (patients.patient_id, patients.site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2, 3, 4, 5, 6))
-          GROUP BY patients.patient_id, patients.site_id
+          SELECT patients.patient_id, 'Defaulted', patients.#{@definition == 'pepfar' ? 'pepfar_defaulter_date' : 'moh_defaulter_date'}, patients.site_id, 5
+          FROM cdr_temp_current_medication AS patients
+          LEFT JOIN cdr_temp_current_state AS cs ON cs.patient_id = patients.patient_id AND cs.site_id = patients.site_id#{' '}
+          WHERE patients.#{@definition == 'pepfar' ? 'pepfar_defaulter_date' : 'moh_defaulter_date'} <= #{end_date}
+          AND (patients.patient_id) NOT IN (SELECT patient_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2, 3, 4))
+          AND patients.site_id = #{location}
           ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), step = VALUES(step)
         SQL
       end
@@ -354,10 +290,10 @@ module MalawiHivProgramReports
       def load_defaulters
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_patient_outcomes PARTITION (p#{location})
-          SELECT patient_id, #{patient_outcome_function('patient_id', 'site_id')}, NULL, site_id, 8
+          SELECT patient_id, #{patient_outcome_function('patient_id', 'site_id')}, NULL, site_id, 6
           FROM cdr_temp_cohort_members
           WHERE date_enrolled <= #{end_date} AND site_id = #{location}
-          AND (patient_id, site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2, 3, 4, 5, 6, 7))
+          AND (patient_id, site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_patient_outcomes PARTITION (p#{location}) WHERE step IN (1, 2, 3, 4, 5))
           ON DUPLICATE KEY UPDATE cum_outcome = VALUES(cum_outcome), outcome_date = VALUES(outcome_date), step = VALUES(step)
         SQL
       end
@@ -367,15 +303,6 @@ module MalawiHivProgramReports
       # ===================================
       #  Function Management Region
       # ===================================
-
-      def current_defaulter_function(sql_column, site_id)
-        case definition
-        when 'moh' then "current_defaulter(#{sql_column}, #{end_date}, #{site_id})"
-        when 'pepfar' then "current_pepfar_defaulter(#{sql_column}, #{end_date}, #{site_id})"
-        else raise "Invalid outcomes definition: #{definition}" # Should never happen but you never know!
-        end
-      end
-
       def patient_outcome_function(sql_column, site_id)
         case definition
         when 'moh' then "patient_outcome(#{sql_column}, #{end_date}, #{site_id})"
@@ -391,8 +318,11 @@ module MalawiHivProgramReports
         create_outcome_table unless check_if_table_exists('cdr_temp_patient_outcomes')
         create_tmp_max_drug_orders_table unless check_if_table_exists('cdr_temp_max_drug_orders')
         create_tmp_min_auto_expire_date unless check_if_table_exists('cdr_temp_min_auto_expire_date')
+        drop_tmp_min_auto_expirte_date unless count_table_columns('cdr_temp_min_auto_expire_date') == 6
         create_cdr_temp_max_patient_state unless check_if_table_exists('cdr_temp_max_patient_state')
-        create_max_patient_appointment_date unless check_if_table_exists('cdr_temp_max_patient_appointment')
+        create_temp_current_state unless check_if_table_exists('cdr_temp_current_state')
+        drop_temp_current_state unless count_table_columns('cdr_temp_current_state') == 7
+        create_temp_current_medication unless check_if_table_exists('cdr_temp_current_medication')
       end
 
       def check_if_table_exists(table_name)
@@ -405,6 +335,16 @@ module MalawiHivProgramReports
         result['count'].to_i.positive?
       end
 
+      def count_table_columns(table_name)
+        result = ActiveRecord::Base.connection.select_one <<~SQL
+          SELECT COUNT(*) AS count
+          FROM INFORMATION_SCHEMA.COLUMNS
+          WHERE table_schema = DATABASE()
+          AND table_name = '#{table_name}'
+        SQL
+        result['count'].to_i
+      end
+
       def create_outcome_table
         ActiveRecord::Base.connection.execute <<~SQL
           CREATE TABLE IF NOT EXISTS cdr_temp_patient_outcomes (
@@ -456,7 +396,10 @@ module MalawiHivProgramReports
           CREATE TABLE IF NOT EXISTS cdr_temp_min_auto_expire_date (
             patient_id INT NOT NULL,
             site_id INT NOT NULL,
+            start_date DATE DEFAULT NULL,
             auto_expire_date DATE DEFAULT NULL,
+            pepfar_defaulter_date DATE DEFAULT NULL,
+            moh_defaulter_date DATE DEFAULT NULL,
             PRIMARY KEY (patient_id, site_id)
           ) ENGINE=InnoDB DEFAULT CHARSET=utf8
           PARTITION BY LIST(site_id) (#{partition_by_site})
@@ -466,7 +409,13 @@ module MalawiHivProgramReports
 
       def create_min_auto_expire_date_indexes
         ActiveRecord::Base.connection.execute <<~SQL
-          CREATE INDEX idx_min_auto_expire_date ON cdr_temp_min_auto_expire_date (auto_expire_date)
+          CREATE INDEX idx_min_auto_expire_date ON temp_min_auto_expire_date (auto_expire_date)
+        SQL
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_min_pepfar ON temp_min_auto_expire_date (pepfar_defaulter_date)
+        SQL
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_min_moh ON temp_min_auto_expire_date (moh_defaulter_date)
         SQL
       end
 
@@ -489,25 +438,84 @@ module MalawiHivProgramReports
         SQL
       end
 
-      def create_max_patient_appointment_date
+      def create_temp_current_state
         ActiveRecord::Base.connection.execute <<~SQL
-          CREATE TABLE IF NOT EXISTS cdr_temp_max_patient_appointment (
+          CREATE TABLE IF NOT EXISTS cdr_temp_current_state(
             patient_id INT NOT NULL,
             site_id INT NOT NULL,
-            appointment_date DATE NOT NULL,
-            PRIMARY KEY (patient_id, site_id)
+            cum_outcome VARCHAR(120) NOT NULL,
+            outcome_date DATE DEFAULT NULL,
+            state INT NOT NULL,
+            outcomes INT NOT NULL,
+            patient_state_id INT NOT NULL,
+            PRIMARY KEY(patient_id, site_id)
+          ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+          PARTITION BY LIST(site_id) (#{partition_by_site})
+        SQL
+        create_current_state_index
+      end
+
+      def create_current_state_index
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_state_name ON cdr_temp_current_state (cum_outcome)
+        SQL
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_state_id ON cdr_temp_current_state (state)
+        SQL
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_state_count ON cdr_temp_current_state (outcomes)
+        SQL
+      end
+
+      def create_temp_current_medication
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE TABLE IF NOT EXISTS cdr_temp_current_medication(
+            patient_id INT NOT NULL,
+            site_id INT NOT NULL,
+            concept_id INT NOT NULL,
+            drug_id INT NOT NULL,
+            daily_dose DECIMAL(32,2) NOT NULL,
+            quantity DECIMAL(32,2) NOT NULL,
+            start_date DATE NOT NULL,
+            pill_count DECIMAL(32,2) NULL,
+            expiry_date DATE NULL,
+            pepfar_defaulter_date DATE NULL,
+            moh_defaulter_date DATE NULL,
+            PRIMARY KEY(patient_id, site_id, drug_id)
           ) ENGINE=InnoDB DEFAULT CHARSET=utf8
           PARTITION BY LIST(site_id) (#{partition_by_site})
         SQL
-        create_max_patient_appointment_date_indexes
+        craete_tmp_current_med_index
       end
 
-      def create_max_patient_appointment_date_indexes
+      def craete_tmp_current_med_index
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_cm_concept ON cdr_temp_current_medication (concept_id)
+        SQL
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_cm_drug ON cdr_temp_current_medication (drug_id)
+        SQL
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_cm_date ON cdr_temp_current_medication (start_date)
+        SQL
+        ActiveRecord::Base.connection.execute <<~SQL
+          CREATE INDEX idx_cm_pepfar ON cdr_temp_current_medication (pepfar_defaulter_date)
+        SQL
         ActiveRecord::Base.connection.execute <<~SQL
-          CREATE INDEX idx_max_patient_appointment_date ON cdr_temp_max_patient_appointment (appointment_date)
+          CREATE INDEX idx_cm_moh ON cdr_temp_current_medication (moh_defaulter_date)
         SQL
       end
 
+      def drop_tmp_min_auto_expirte_date
+        ActiveRecord::Base.connection.execute 'DROP TABLE cdr_temp_min_auto_expire_date'
+        create_tmp_min_auto_expire_date
+      end
+
+      def drop_temp_current_state
+        ActiveRecord::Base.connection.execute 'DROP TABLE cdr_temp_current_state'
+        create_temp_current_state
+      end
+
       def update_steps
         ActiveRecord::Base.connection.execute <<~SQL
           UPDATE cdr_temp_patient_outcomes SET step = 0 WHERE step > 0 AND site_id = #{location}
@@ -525,15 +533,15 @@ module MalawiHivProgramReports
           ActiveRecord::Base.connection.execute('TRUNCATE cdr_temp_max_drug_orders')
           ActiveRecord::Base.connection.execute('TRUNCATE cdr_temp_min_auto_expire_date')
           ActiveRecord::Base.connection.execute('TRUNCATE cdr_temp_max_patient_state')
-          ActiveRecord::Base.connection.execute('TRUNCATE cdr_temp_max_patient_appointment')
-          ActiveRecord::Base.connection.execute('TRUNCATE cdr_temp_cohort_status')
+          ActiveRecord::Base.connection.execute('TRUNCATE cdr_temp_current_state')
+          ActiveRecord::Base.connection.execute('TRUNCATE cdr_temp_current_medication')
         else
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_patient_outcomes WHERE site_id = #{location}")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_max_drug_orders WHERE site_id = #{location}")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_min_auto_expire_date WHERE site_id = #{location}")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_max_patient_state WHERE site_id = #{location}")
-          ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_max_patient_appointment WHERE site_id = #{location}")
-          ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_cohort_status WHERE site_id = #{location}")
+          ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_current_state WHERE site_id = #{location}")
+          ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_current_medication WHERE site_id = #{location}")
         end
       end
     end

data/lib/malawi_hiv_program_reports/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MalawiHivProgramReports
-  VERSION = '1.0.21'
+  VERSION = '1.0.23'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: malawi_hiv_program_reports
 version: !ruby/object:Gem::Version
-  version: 1.0.21
+  version: 1.0.23
 platform: ruby
 authors:
 - Roy Chanunkha
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-06 00:00:00.000000000 Z
+date: 2024-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -174,7 +174,7 @@ licenses:
 metadata:
   source_code_uri: https://github.com/EGPAFMalawiHIS/malawi_hiv_program_reports
   rubygems_mfa_required: 'true'
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -189,8 +189,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
-signing_key: 
+rubygems_version: 3.5.6
+signing_key:
 specification_version: 4
 summary: Malawi HIV PROGRAM Reports for Ruby on Rails
 test_files: []