malawi_hiv_program_reports 1.0.25 → 1.0.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 062d15881512b1cc583e750ad84ad6d48c5ae101bfe115faa29661f92a5a4c5f
-  data.tar.gz: e8b0ee4083537c6880f2ac0f362e647a02c670cf73f50af6db92b2b31cf1855a
+  metadata.gz: 53d6565fca96ee2021dfe480bdede7d60e308abc3988c209c86658e56d66baa4
+  data.tar.gz: c8f56f53589f21725e00c4afe769ffbc330f6a2a687455e6fd43bc128657eb8f
 SHA512:
-  metadata.gz: 9e262010a9b6b969e1cba70790c82ff8b4da89f45d3b902307ce5a9f9a9364f51e9f32077d9a7c962b3505f33e7f93834b424d233c9d1e4ea5c128a0db6b7304
-  data.tar.gz: 7663c5ba36fd8346e41aa41b432951fc286da066c89a63f7966cd28be6a843247d66ed1869e31414793673dd2db3cba1c972d17666924833fa8de40916901c41
+  metadata.gz: b9909ffb6013589e67b6ccce8f7833308270cedb69c74d7ac77a203b2870475e69bbef0c635d88ad51d1095958670313b26c862dc16407358aa448a3eaf1378e
+  data.tar.gz: a08774fa4213673af1cfa0efa728059a52bbe6dcc0f0dd70174ebaddc68581ba25c50d53a56946c474ed99bd60234762b67c8e57b77f0d94eea8a2477790f27a

data/app/services/malawi_hiv_program_reports/moh/cumulative_cohort.rb

@@ -17,13 +17,17 @@ module MalawiHivProgramReports
         @rebuild = kwargs[:rebuild]&.casecmp?('true')
         locations = kwargs[:locations]
         @locations = locations.present? ? locations.split(',') : []
-        @definition = kwargs[:definition]
+        @definition = kwargs[:definition] || 'pepfar'
+
+        definition = @definition.downcase
+        raise ArgumentError, "Invalid outcomes definition: #{definition}" unless %w[moh pepfar].include?(definition)
       end
 
       def find_report
         start_time = Time.now
         prepare_tables
         clear_tables if rebuild
+        clear_cohort_status unless rebuild
         process_thread
         end_time = Time.now
         time_in_minutes = ((end_time - start_time) / 60).round(2)
@@ -60,7 +64,8 @@ module MalawiHivProgramReports
               ActiveRecord::Base.connection_pool.with_connection do
                 process_data loc
               rescue StandardError => e
-                Rails.logger.error("Error processing location #{loc}: #{e.message}")
+                Rails.logger.info("Error processing location #{loc}: #{e.message}")
+                Rails.logger.info(e.backtrace.join("\n"))
                 save_incomplete_site(location: loc, time_taken: 0)
               end
             end
@@ -80,8 +85,8 @@ module MalawiHivProgramReports
         potential_cohort_members location
         reason_for_starting_art location
         cohort_members location
-        outcome = MalawiHivProgramReports::Moh::CumulativeOutcome.new(end_date:, location:, definition:, rebuild:)
-        outcome.find_report
+        outcome = MalawiHivProgramReports::Moh::CumulativeOutcome.new(end_date:, location:, definition:, rebuild:, start_date:)
+        rebuild ? outcome.find_report : outcome.update_outcomes_by_definition
         end_time = Time.now
         time_taken = ((end_time - start_time) / 60).round(2)
         save_completed_site(location:, time_taken:)
@@ -260,7 +265,7 @@ module MalawiHivProgramReports
           INSERT INTO cdr_temp_cohort_members PARTITION (p#{location})
           SELECT
               pcm.patient_id,
-              pcm.site_id,
+              #{location},
               pcm.birthdate,
               pcm.birthdate_estimated,
               pcm.death_date,
@@ -271,11 +276,10 @@ module MalawiHivProgramReports
               IF(pcm.birthdate IS NOT NULL, TIMESTAMPDIFF(YEAR, pcm.birthdate,  COALESCE(ti.value_datetime, mdo.start_date)), NULL) AS age_at_initiation,
               IF(pcm.birthdate IS NOT NULL, TIMESTAMPDIFF(DAY, pcm.birthdate,  COALESCE(ti.value_datetime, mdo.start_date)), NULL) AS age_in_days,
               rfsa.reason_for_starting_art
-          FROM cdr_temp_potential_cohort_members pcm
-          INNER JOIN cdr_temp_min_drug_orders mdo ON mdo.patient_id = pcm.patient_id AND mdo.site_id = pcm.site_id
-          INNER JOIN cdr_reason_for_starting_art rfsa ON rfsa.patient_id = pcm.patient_id AND rfsa.site_id = pcm.site_id
-          LEFT JOIN cdr_temp_transfer_ins ti ON ti.patient_id = pcm.patient_id AND ti.site_id = pcm.site_id
-          WHERE pcm.site_id = #{location}
+          FROM cdr_temp_potential_cohort_members PARTITION (p#{location}) pcm
+          INNER JOIN cdr_temp_min_drug_orders PARTITION (p#{location}) mdo ON mdo.patient_id = pcm.patient_id
+          INNER JOIN cdr_reason_for_starting_art PARTITION (p#{location}) AS rfsa ON rfsa.patient_id = pcm.patient_id
+          LEFT JOIN cdr_temp_transfer_ins PARTITION (p#{location}) ti ON ti.patient_id = pcm.patient_id
           HAVING reason_for_starting_art IS NOT NULL
             ON DUPLICATE KEY UPDATE birthdate = VALUES(birthdate), birthdate_estimated = VALUES(birthdate_estimated), death_date = VALUES(death_date),
             gender = VALUES(gender), date_enrolled = VALUES(date_enrolled), earliest_start_date = VALUES(earliest_start_date),
@@ -289,19 +293,18 @@ module MalawiHivProgramReports
           INSERT INTO cdr_temp_potential_cohort_members PARTITION (p#{location})
           SELECT
               pp.patient_id,
-              pp.site_id,
+              #{location},
               p.birthdate,
               p.birthdate_estimated,
               p.death_date,
               LEFT(p.gender, 1) gender
-          FROM patient_program pp
-          INNER JOIN person p ON p.person_id = pp.patient_id AND p.site_id = pp.site_id AND p.voided = 0
-          INNER JOIN patient_state ps ON ps.patient_program_id = pp.patient_program_id AND ps.site_id = pp.site_id
+          FROM patient_program PARTITION (p#{location}) AS pp
+          INNER JOIN person PARTITION (p#{location}) AS p ON p.person_id = pp.patient_id AND p.voided = 0
+          INNER JOIN patient_state PARTITION (p#{location}) AS ps ON ps.patient_program_id = pp.patient_program_id
               AND ps.voided = 0 AND ps.state = 7 AND ps.start_date IS NOT NULL -- 7 is On antiretrovirals
               AND ps.start_date < (DATE(#{end_date}) + INTERVAL 1 DAY)
-          WHERE pp.program_id = 1 AND pp.voided = 0 AND (pp.patient_id, pp.site_id) NOT IN (SELECT patient_id, site_id FROM cdr_temp_external_clients)
-          AND pp.site_id = #{location}
-          GROUP BY pp.patient_id, pp.site_id
+          WHERE pp.program_id = 1 AND pp.voided = 0 AND (pp.patient_id) NOT IN (SELECT patient_id FROM cdr_temp_external_clients PARTITION (p#{location}))
+          GROUP BY pp.patient_id
           ON DUPLICATE KEY UPDATE birthdate = VALUES(birthdate), birthdate_estimated = VALUES(birthdate_estimated), death_date = VALUES(death_date),
             gender = VALUES(gender)
         SQL
@@ -310,24 +313,24 @@ module MalawiHivProgramReports
       def min_drug_orders(location)
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_min_drug_orders PARTITION (p#{location})
-          SELECT o.patient_id, o.site_id, DATE(MIN(o.start_date)) start_date
-          FROM orders o
-          INNER JOIN drug_order do ON do.order_id = o.order_id AND do.site_id = o.site_id AND do.quantity > 0
+          SELECT o.patient_id, #{location}, DATE(MIN(o.start_date)) start_date
+          FROM orders PARTITION (p#{location}) AS o
+          INNER JOIN drug_order PARTITION (p#{location}) AS do ON do.order_id = o.order_id AND do.quantity > 0
           LEFT JOIN (
-            SELECT o.person_id patient_id, o.site_id, DATE(MIN(o.obs_datetime)) registered_date
-            FROM obs o
-            INNER JOIN cdr_other_patient_types other ON other.patient_id = o.person_id AND other.site_id = o.site_id
+            SELECT o.person_id patient_id, #{location}, DATE(MIN(o.obs_datetime)) registered_date
+            FROM obs PARTITION (p#{location}) AS o
+            INNER JOIN cdr_other_patient_types PARTITION (p#{location}) AS other ON other.patient_id = o.person_id
             WHERE o.concept_id = 3289 -- Type of patient
                 AND o.value_coded = 7572 -- New patient
                 AND o.voided = 0
                 AND o.obs_datetime < (DATE(#{end_date}) + INTERVAL 1 DAY)
-            GROUP BY o.person_id, o.site_id
-          ) np ON np.patient_id = o.patient_id AND np.site_id = o.site_id
+            GROUP BY o.person_id
+          ) np ON np.patient_id = o.patient_id
           WHERE o.voided = 0 AND o.start_date < (DATE(#{end_date}) + INTERVAL 1 DAY)
             AND o.start_date > COALESCE(np.registered_date, DATE('1900-01-01'))
             AND o.concept_id IN (SELECT concept_id FROM concept_set WHERE concept_set = 1085) -- 1085 is ARV DRUGS
-            AND o.order_type_id = 1 AND o.site_id = #{location}
-          GROUP BY o.patient_id, o.site_id
+            AND o.order_type_id = 1
+          GROUP BY o.patient_id
           ON DUPLICATE KEY UPDATE start_date = VALUES(start_date)
         SQL
       end
@@ -335,15 +338,15 @@ module MalawiHivProgramReports
       def transfer_ins(location)
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_transfer_ins PARTITION (p#{location})
-          SELECT o.person_id, o.site_id, DATE(MIN(o.value_datetime)) value_datetime
-          FROM obs o
-          INNER JOIN encounter e ON e.patient_id = o.person_id AND e.site_id = o.site_id AND e.encounter_id = o.encounter_id
-           AND e.program_id = 1 AND e.encounter_datetime < (DATE(#{end_date}) + INTERVAL 1 DAY)
+          SELECT o.person_id, #{location}, DATE(MIN(o.value_datetime)) value_datetime
+          FROM obs PARTITION (p#{location}) AS o
+          INNER JOIN encounter PARTITION (p#{location}) AS e ON e.patient_id = o.person_id AND e.encounter_id = o.encounter_id
+           AND e.program_id = 1 -- HIV Program
+           AND e.encounter_datetime < (DATE(#{end_date}) + INTERVAL 1 DAY)
            AND e.encounter_type = 9 -- HIV CLINIC REGISTRATION
            AND e.voided = 0
           WHERE o.concept_id = 2516 AND o.voided = 0 AND o.obs_datetime < (DATE(#{end_date}) + INTERVAL 1 DAY) -- 2516 is Date antiretrovirals started
-          AND o.site_id = #{location}
-          GROUP BY o.person_id, o.site_id
+          GROUP BY o.person_id
           ON DUPLICATE KEY UPDATE value_datetime = VALUES(value_datetime)
         SQL
       end
@@ -351,24 +354,21 @@ module MalawiHivProgramReports
       def external_clients(location)
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_temp_external_clients PARTITION (p#{location})
-          SELECT e.patient_id, e.site_id, GROUP_CONCAT(DISTINCT(patient_type.value_coded)) AS patient_types, clinic_registration.encounter_id
-          FROM patient_program as e
-          INNER JOIN obs AS patient_type ON patient_type.person_id = e.patient_id
-            AND patient_type.site_id = e.site_id
+          SELECT e.patient_id, #{location}, GROUP_CONCAT(DISTINCT(patient_type.value_coded)) AS patient_types, clinic_registration.encounter_id
+          FROM patient_program PARTITION (p#{location}) AS e
+          INNER JOIN obs PARTITION (p#{location}) AS patient_type ON patient_type.person_id = e.patient_id
             AND patient_type.voided = 0
             AND patient_type.concept_id = 3289 -- Type of patient
             AND patient_type.obs_datetime < DATE(#{end_date}) + INTERVAL 1 DAY
-          LEFT JOIN encounter as clinic_registration ON clinic_registration.patient_id  = e.patient_id
-            AND clinic_registration.site_id = e.site_id
-            AND clinic_registration.program_id = 1
+          LEFT JOIN encounter PARTITION (p#{location}) AS clinic_registration ON clinic_registration.patient_id  = e.patient_id
+            AND clinic_registration.program_id = 1 -- HIV PROGRAM
             AND clinic_registration.encounter_type = 9 -- HIV CLINIC REGISTRATION
             AND clinic_registration.encounter_datetime < DATE(#{end_date}) + INTERVAL 1 DAY
             AND clinic_registration.voided = 0
           WHERE e.program_id = 1 -- HIV program
             AND e.voided = 0
-            AND e.site_id = #{location}
             -- AND clinic_registration.encounter_id IS NOT NULL -- bone of contention
-          GROUP BY e.patient_id, e.site_id
+          GROUP BY e.patient_id
           HAVING FIND_IN_SET('7572', patient_types) = 0 AND encounter_id IS NULL
           ON DUPLICATE KEY UPDATE patient_types = VALUES(patient_types)
         SQL
@@ -377,14 +377,13 @@ module MalawiHivProgramReports
       def cdr_other_patient_types(location)
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_other_patient_types PARTITION (p#{location})
-          SELECT o.person_id, o.site_id
-          FROM obs o
+          SELECT o.person_id, #{location}
+          FROM obs PARTITION (p#{location}) AS o
           WHERE o.concept_id = 3289 -- Type of patient
               AND o.value_coded != 7572 -- New patient
               AND o.voided = 0
               AND o.obs_datetime < (DATE(#{end_date}) + INTERVAL 1 DAY)
-              AND o.site_id = #{location}
-          GROUP BY o.person_id, o.site_id
+          GROUP BY o.person_id
           -- on duplicate just ignore
           ON DUPLICATE KEY UPDATE site_id = VALUES(site_id)
         SQL
@@ -393,14 +392,15 @@ module MalawiHivProgramReports
       def reason_for_starting_art(location)
         ActiveRecord::Base.connection.execute <<~SQL
           INSERT INTO cdr_reason_for_starting_art PARTITION (p#{location})
-          SELECT a.person_id, a.site_id, a.value_coded
-          FROM obs a
-          INNER JOIN cdr_temp_potential_cohort_members ct ON ct.patient_id = a.person_id AND ct.site_id = a.site_id
-          LEFT OUTER JOIN obs b ON a.person_id = b.person_id AND a.site_id = b.site_id AND b.concept_id  = a.concept_id
-          AND b.concept_id  = 7563
-          AND a.obs_datetime < b.obs_datetime AND b.voided = 0 AND b.obs_datetime < DATE(#{end_date}) + INTERVAL 1 DAY AND a.obs_datetime < DATE(#{end_date}) + INTERVAL 1 DAY
-          WHERE b.obs_id IS NULL AND a.concept_id = 7563 AND a.voided = 0 AND a.site_id = #{location}
-          GROUP BY a.person_id, a.site_id
+          SELECT a.person_id, #{location}, a.value_coded
+          FROM obs PARTITION (p#{location}) a
+          INNER JOIN cdr_temp_potential_cohort_members PARTITION (p#{location}) AS ct ON ct.patient_id = a.person_id
+          LEFT OUTER JOIN obs PARTITION (p#{location}) AS b ON a.person_id = b.person_id AND b.concept_id  = a.concept_id
+            AND b.concept_id  = 7563
+            AND a.obs_datetime < b.obs_datetime AND b.voided = 0
+            AND b.obs_datetime < DATE(#{end_date}) + INTERVAL 1 DAY AND a.obs_datetime < DATE(#{end_date}) + INTERVAL 1 DAY
+          WHERE b.obs_id IS NULL AND a.concept_id = 7563 AND a.voided = 0
+          GROUP BY a.person_id
           ON DUPLICATE KEY UPDATE reason_for_starting_art = VALUES(reason_for_starting_art)
         SQL
       end
@@ -417,7 +417,6 @@ module MalawiHivProgramReports
           ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_temp_cohort_members')
           ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_temp_external_clients')
           ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_reason_for_starting_art')
-          ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_temp_cohort_status')
         else
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_other_patient_types WHERE site_id IN (#{locations.join(',')})")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_potential_cohort_members WHERE site_id IN (#{locations.join(',')})")
@@ -426,6 +425,14 @@ module MalawiHivProgramReports
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_cohort_members WHERE site_id IN (#{locations.join(',')})")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_external_clients WHERE site_id IN (#{locations.join(',')})")
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_reason_for_starting_art WHERE site_id IN (#{locations.join(',')})")
+        end
+        clear_cohort_status
+      end
+      
+      def clear_cohort_status
+        if locations.empty?
+          ActiveRecord::Base.connection.execute('TRUNCATE TABLE cdr_temp_cohort_status')
+        else
           ActiveRecord::Base.connection.execute("DELETE FROM cdr_temp_cohort_status WHERE site_id IN (#{locations.join(',')})")
         end
       end