openmrs_data_sanitizer 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa93214174e11f30a2ac4a89f33babc9fea061e7fed3765f2679241ec28b8955
-  data.tar.gz: d70e7d5a57cdac1fe270a32e9bf37a5b523d7b1b9ca9524d6ce65d4e8a0ee820
+  metadata.gz: 871a8c260924d6911e5ed550ad158e3914d10eab5a93fc473bb37d56bcbcc641
+  data.tar.gz: 9af6e3f50d1beec93f505cedacaf0722bc66d4dacb69dbc3c6104166a7c333c5
 SHA512:
-  metadata.gz: d2fe7b43e4e5909e814ee99066e7c256be2f87a7cf42bdd6d8be90ebee1b81bbdfeea6c845b5a59dda2d3dbceebcf1edc6c9e61ec11c12d8b07d4ff447d73d3c
-  data.tar.gz: aad5a77b0677e7d7043ae61bff9609dfdc323768abefbf9a6a5d53e7a37b60be00e791896fb343a255a5cc03cee90fc94922bde427d2743ac6a297f1e333e2a5
+  metadata.gz: 8117cb24ce418d40e297585b23d0d5f675af86290067a6d484dee81fe0a6a85fc3f3a62b2c383fc55983120269ea702182906368b9ddd795737861183e9253ee
+  data.tar.gz: 8a145dc8da6d99a3ec3ae2287a5a44927a187c580b405d109d639f0385e340e9fa605512e06ba94f6ebf48e4b07695dddbb91bd8bbca78d7ff7e13e1b64e913a

data/app/services/openmrs_data_sanitizer/data_aggregator.rb

@@ -95,7 +95,7 @@ module OpenmrsDataSanitizer
         drop_data_sanitizer_encounter_dates_table
         create_data_sanitizer_encounter_dates_table
       end
-      if count_table_columns('data_sanitizer_patient_outcomes') != 5
+      if count_table_columns('data_sanitizer_patient_outcomes') != 6
         drop_data_sanitizer_patient_outcomes_table
         create_data_sanitizer_patient_outcomes_table
       end
@@ -108,17 +108,22 @@ module OpenmrsDataSanitizer
         drop_data_sanitizer_patient_reason_for_starting_table
         create_data_sanitizer_patient_reason_for_starting_table
       end
-      if count_table_columns('data_sanitizer_patient_detailed_side_effects') != 21
+      if count_table_columns('data_sanitizer_patient_detailed_side_effects') != 24
         drop_data_sanitizer_patient_detailed_side_effects_table
         create_data_sanitizer_patient_detailed_side_effects_table
       end
-      return unless count_table_columns('data_sanitizer_visits') != 35
+      if count_table_columns('data_sanitizer_patients') != 8
+        drop_data_sanitizer_patients_table
+        create_data_sanitizer_patients_table
+      end
+      return unless count_table_columns('data_sanitizer_visits') != 37
 
       drop_data_sanitizer_visits_table
       create_data_sanitizer_visits_table
     end
 
     def process_data
+      insert_data_sanitizer_patients
       insert_data_sanitizer_patients_height
       insert_data_sanitizer_patients_weight
       insert_data_sanitizer_patient_present
@@ -144,7 +149,7 @@ module OpenmrsDataSanitizer
                         data_sanitizer_side_effect data_sanitizer_adherence data_sanitizer_appointment
                         data_sanitizer_arv_medication data_sanitizer_arv_orders data_sanitizer_encounter_dates
                         data_sanitizer_patient_outcomes data_sanitizer_patient_who_stage data_sanitizer_patient_reason_for_starting
-                        data_sanitizer_patient_detailed_side_effects data_sanitizer_visits ]
+                        data_sanitizer_patient_detailed_side_effects data_sanitizer_visits data_sanitizer_patients]
       table_names.each do |table_name|
         ActiveRecord::Base.connection.execute("TRUNCATE TABLE #{schema}.#{table_name}")
       end
@@ -415,6 +420,21 @@ module OpenmrsDataSanitizer
       SQL
       create_data_sanitizer_visits_index
     end
+
+    def create_data_sanitizer_patients_table
+      ActiveRecord::Base.connection.execute <<~SQL
+        CREATE TABLE IF NOT EXISTS #{schema}.data_sanitizer_patients (
+          patient_id INT PRIMARY KEY,
+          given_name VARCHAR(100),
+          middle_name VARCHAR(100),
+          family_name VARCHAR(100),
+          gender VARCHAR(1),
+          birthdate DATE,
+          phone_number VARCHAR(100),
+          village_name VARCHAR(100)
+        );
+      SQL
+    end
     # rubocop:enable Metrics/MethodLength
 
     # ========================
@@ -527,6 +547,12 @@ module OpenmrsDataSanitizer
     # DATABASE TABLE DELETE METHODS
     # ========================
 
+    def drop_data_sanitizer_patients_table
+      ActiveRecord::Base.connection.execute <<~SQL
+        DROP TABLE IF EXISTS #{schema}.data_sanitizer_patients;
+      SQL
+    end
+
     def drop_data_sanitizer_patients_height_table
       ActiveRecord::Base.connection.execute <<~SQL
         DROP TABLE IF EXISTS #{schema}.data_sanitizer_patients_height;
@@ -634,19 +660,28 @@ module OpenmrsDataSanitizer
     # ========================
 
     # rubocop:disable Metrics/MethodLength
+    def insert_data_sanitizer_patients
+      ActiveRecord::Base.connection.execute <<~SQL
+        INSERT INTO #{schema}.data_sanitizer_patients (patient_id, given_name, middle_name, family_name, gender, birthdate, phone_number, village_name)
+        SELECT p.person_id, n.given_name, n.middle_name, n.family_name, IF(p.gender is NULL, 'U', LEFT(p.gender,1)), p.birthdate, phone.value, village.value
+        FROM #{schema}.person p
+        INNER JOIN #{schema}.patient ON patient.patient_id = p.person_id AND p.voided = 0
+        LEFT JOIN #{schema}.person_name n ON n.person_id = p.person_id AND n.voided = 0
+        LEFT JOIN #{schema}.person_attribute phone ON phone.person_id = p.person_id AND phone.voided = 0 AND phone.person_attribute_type_id = 12
+        LEFT JOIN #{schema}.person_attribute village ON village.person_id = p.person_id AND village.voided = 0 AND village.person_attribute_type_id = 19
+        WHERE p.voided = 0
+        GROUP BY p.person_id;
+      SQL
+    end
+
     def insert_data_sanitizer_patients_height
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_patients_height (patient_id, height, visit_date)
         SELECT e.patient_id, ROUND(obs.value_numeric), DATE(obs.obs_datetime) visit_date
         FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs ON e.patient_id = obs.person_id
-        AND e.voided = 0
-        AND e.encounter_type = 6
-        AND e.program_id = 1
-        AND obs.voided = 0
-        AND obs.concept_id = 5090 -- Height concept_id
-        GROUP BY obs.person_id, DATE(obs.obs_datetime)
-        ORDER BY obs_datetime DESC;
+        INNER JOIN #{schema}.obs ON e.encounter_id = obs.encounter_id AND obs.voided = 0 AND obs.concept_id = 5090 -- Height concept_id
+        WHERE e.voided = 0 and e.encounter_type = 6 and e.program_id = 1
+        GROUP BY obs.person_id, DATE(obs.obs_datetime);
       SQL
     end
 
@@ -655,14 +690,11 @@ module OpenmrsDataSanitizer
         INSERT INTO #{schema}.data_sanitizer_patients_weight (patient_id, weight, visit_date)
         SELECT e.patient_id, ROUND(obs.value_numeric), DATE(obs.obs_datetime) visit_date
         FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs ON e.patient_id = obs.person_id
-        AND e.voided = 0
-        AND e.program_id = 1
-        AND e.encounter_type = 6
-        AND obs.voided = 0
-        AND obs.concept_id = 5089
-        GROUP BY obs.person_id, DATE(obs.obs_datetime)
-        ORDER BY obs_datetime DESC;
+        INNER JOIN #{schema}.obs ON e.encounter_id = obs.encounter_id
+          AND obs.voided = 0
+          AND obs.concept_id = 5089
+        WHERE e.voided = 0 and e.encounter_type = 6 and e.program_id = 1
+        GROUP BY obs.person_id, DATE(obs.obs_datetime);
       SQL
     end
 
@@ -673,14 +705,11 @@ module OpenmrsDataSanitizer
         (CASE WHEN obs.value_coded = 1065 THEN 'YES' ELSE 'NO' END) AS patient_present,
         DATE(obs.obs_datetime) visit_date
         FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs ON e.patient_id = obs.person_id
-        AND e.voided = 0
-        AND e.encounter_type = 51
-        AND e.program_id = 1
+        INNER JOIN #{schema}.obs ON e.encounter_id = obs.encounter_id
         AND obs.voided = 0
         AND obs.concept_id = 1805
-        GROUP BY obs.person_id, DATE(obs.obs_datetime)
-        ORDER BY obs_datetime DESC;
+        WHERE e.voided = 0 and e.encounter_type = 51 and e.program_id = 1
+        GROUP BY obs.person_id, DATE(obs.obs_datetime);
       SQL
     end
 
@@ -691,16 +720,13 @@ module OpenmrsDataSanitizer
         (CASE WHEN obs.value_coded = 1065 THEN 'YES' ELSE 'NO' END) AS guardian_present,
         DATE(obs.obs_datetime) visit_date
         FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs ON e.patient_id = obs.person_id
-        AND e.voided = 0
-        AND e.encounter_type = 51
-        AND e.program_id = 1
-        AND obs.voided = 0
-        AND obs.concept_id = 6794
+        INNER JOIN #{schema}.obs ON e.encounter_id = obs.encounter_id
+          AND obs.voided = 0
+          AND obs.concept_id = 6794
         INNER JOIN #{schema}.person ON person.person_id = obs.person_id
-        AND person.gender IN('Female','F')
-        GROUP BY obs.person_id, DATE(obs.obs_datetime)
-        ORDER BY obs_datetime DESC;
+          AND person.gender IN('Female','F')
+        WHERE e.voided = 0 and e.encounter_type = 51 and e.program_id = 1
+        GROUP BY obs.person_id, DATE(obs.obs_datetime);
       SQL
     end
 
@@ -711,16 +737,13 @@ module OpenmrsDataSanitizer
         (CASE WHEN obs.value_coded = 1065 THEN 'YES' ELSE 'NO' END) AS pregnant_status,
         DATE(obs.obs_datetime) visit_date
         FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs ON e.patient_id = obs.person_id
-        AND e.voided = 0
-        AND e.program_id = 1
-        AND e.encounter_type = 53
-        AND obs.voided = 0
-        AND obs.concept_id IN(6131, 1755)
+        INNER JOIN #{schema}.obs ON e.encounter_id = obs.encounter_id
+          AND obs.voided = 0
+          AND obs.concept_id IN(6131, 1755)
         INNER JOIN #{schema}.person ON person.person_id = obs.person_id
-        AND person.gender IN('Female','F')
-        GROUP BY obs.person_id, DATE(obs.obs_datetime)
-        ORDER BY obs_datetime DESC;
+          AND person.gender IN('Female','F')
+        WHERE e.voided = 0 and e.encounter_type = 53 and e.program_id = 1
+        GROUP BY obs.person_id, DATE(obs.obs_datetime);
       SQL
     end
 
@@ -731,34 +754,28 @@ module OpenmrsDataSanitizer
         (CASE WHEN obs.value_coded = 1065 THEN 'YES' ELSE 'NO' END) AS breastfeeding_status,
         DATE(obs.obs_datetime) visit_date
         FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs ON e.patient_id = obs.person_id
-        AND e.voided = 0
-        AND e.program_id = 1
-        AND e.encounter_type = 53
-        AND obs.voided = 0
-        AND obs.concept_id IN(834,5632, 8039,7965)
-        GROUP BY obs.person_id, DATE(obs.obs_datetime)
-        ORDER BY obs_datetime DESC;
+        INNER JOIN #{schema}.obs ON e.encounter_id = obs.encounter_id
+          AND obs.voided = 0
+          AND obs.concept_id IN(834,5632, 8039,7965)
+        WHERE e.voided = 0 and e.encounter_type = 53 and e.program_id = 1
+        GROUP BY obs.person_id, DATE(obs.obs_datetime);
       SQL
     end
 
     def insert_data_sanitizer_side_effect
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_side_effect (patient_id, visit_date, concept_id, value_coded, available)
-        SELECT t.person_id,
-        DATE(t.obs_datetime) visit_date,
-        t2.concept_id,
-        t2.value_coded,
+        SELECT t.person_id,DATE(t.obs_datetime) visit_date,t2.concept_id,t2.value_coded,
         (CASE WHEN t2.value_coded = 1065 THEN 'YES' ELSE 'NO' END) available
         FROM #{schema}.obs t
         INNER JOIN #{schema}.obs t2 ON t.concept_id = 7755
-        AND t.obs_id = t2.obs_group_id
-        AND t.voided = 0
-        AND t2.voided = 0
+          AND t.obs_id = t2.obs_group_id
+          AND t.voided = 0
+          AND t2.voided = 0
         INNER JOIN #{schema}.encounter e ON e.encounter_id = t.encounter_id
-        AND e.program_id = 1
-        AND e.encounter_type = 53
-        AND e.voided = 0
+          AND e.program_id = 1
+          AND e.encounter_type = 53
+          AND e.voided = 0
         GROUP BY t.person_id, DATE(t.obs_datetime), t.value_coded, t2.value_coded;
       SQL
     end
@@ -766,61 +783,46 @@ module OpenmrsDataSanitizer
     def insert_data_sanitizer_adherence
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_adherence (patient_id, visit_date, adherence)
-        SELECT o.person_id,
-          DATE(o.visit_date) visit_date,
+        SELECT o.person_id, DATE(o.visit_date) visit_date,
           ROUND(MAX(CASE WHEN (REPLACE(adherence, '%', '')) BETWEEN 95 AND 105 THEN (REPLACE(adherence, '%', '')) ELSE 0 END)) AS adherence
         FROM (
           SELECT obs.person_id, DATE(obs_datetime) AS visit_date, IF(value_numeric REGEXP '^[+-]?[0-9]+(\.[0-9]+)?%?$', CAST(REPLACE(value_numeric, '%', '') AS DECIMAL(10, 2)), 0) AS adherence
           FROM #{schema}.obs
-          INNER JOIN #{schema}.encounter e ON e.encounter_type = 68
-            AND e.encounter_id = obs.encounter_id
+          INNER JOIN #{schema}.encounter e ON e.encounter_id = obs.encounter_id 
+            AND e.encounter_type = 68
             AND e.program_id = 1
           WHERE obs.voided = 0
             AND obs.concept_id = 6987 -- Drug adherence concept_id
             AND (obs.value_numeric IS NOT NULL OR obs.value_text IS NOT NULL)
         ) AS o
         GROUP BY o.person_id, o.visit_date
-        ORDER BY o.person_id, o.visit_date;
       SQL
     end
 
     def insert_data_sanitizer_appointment
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_appointment (patient_id, value_datetime, visit_date)
-        SELECT e.patient_id,
-        DATE(value_datetime),
-        DATE(obs.obs_datetime) visit_date
+        SELECT e.patient_id, DATE(value_datetime),DATE(obs.obs_datetime) visit_date
         FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs ON e.patient_id = obs.person_id
-        AND e.voided = 0
-        AND e.encounter_type = 7
-        AND e.program_id = 1
-        AND obs.voided = 0
-        AND obs.concept_id = 5096
-        GROUP BY obs.person_id, DATE(obs.obs_datetime)
-        ORDER BY obs_datetime DESC;
+        INNER JOIN #{schema}.obs ON e.encounter_id = obs.encounter_id 
+          AND obs.voided = 0
+          AND obs.concept_id = 5096
+        WHERE e.voided = 0 and e.encounter_type = 7 and e.program_id = 1
+        GROUP BY obs.person_id, DATE(obs.obs_datetime);
       SQL
     end
 
     def insert_data_sanitizer_arv_medication
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_arv_medication (patient_id, order_id, visit_date, drug_runout_date, num_unique_arvs, unique_arvs_given, drug_names)
-        SELECT e.patient_id, o.order_id ,
-        DATE(o.start_date) AS visit_date,
-        DATE(MIN(o.auto_expire_date)) AS drug_runout_date,
-        COUNT(DISTINCT d.drug_id) AS num_unique_arvs,
-        GROUP_CONCAT(DISTINCT d.drug_id SEPARATOR ', ') AS unique_arvs_given,
-        GROUP_CONCAT(DISTINCT d.name SEPARATOR ', ') AS drug_names
+        SELECT e.patient_id, o.order_id, DATE(o.start_date) AS visit_date, DATE(MIN(o.auto_expire_date)) AS drug_runout_date,
+          COUNT(DISTINCT d.drug_id) AS num_unique_arvs,GROUP_CONCAT(DISTINCT d.drug_id SEPARATOR ', ') AS unique_arvs_given,
+          GROUP_CONCAT(DISTINCT d.name SEPARATOR ', ') AS drug_names
         FROM #{schema}.orders o
-        INNER JOIN #{schema}.drug_order da ON da.order_id = o.order_id
-        AND o.voided = 0
-        AND da.quantity > 0
+        INNER JOIN #{schema}.drug_order da ON da.order_id = o.order_id AND o.voided = 0 AND da.quantity > 0
         INNER JOIN #{schema}.drug d ON d.drug_id = da.drug_inventory_id
-        INNER JOIN #{schema}.obs ON obs.order_id = o.order_id
-        AND obs.voided = 0
-        INNER JOIN #{schema}.encounter e ON e.program_id = 1
-        AND e.encounter_id = obs.encounter_id
-        AND e.encounter_type = 54
+        INNER JOIN #{schema}.obs ON obs.order_id = o.order_id AND obs.voided = 0
+        INNER JOIN #{schema}.encounter e ON e.program_id = 1 AND e.encounter_id = obs.encounter_id AND e.encounter_type = 54
         INNER JOIN #{schema}.arv_drug ON arv_drug.drug_id = d.drug_id
         GROUP BY obs.person_id, o.order_id, DATE(o.start_date);
       SQL
@@ -829,10 +831,9 @@ module OpenmrsDataSanitizer
     def insert_data_sanitizer_arv_orders
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_arv_orders(patient_id, order_id, encounter_id, visit_date, drug_id, drug_name, units, equivalent_daily_dose, instructions, start_date, runout_date, quantity)
-        SELECT
-          m.patient_id, o.order_id, orders.encounter_id , m.visit_date,
-            d.drug_id, d.name, d.units, o.equivalent_daily_dose, orders.instructions,
-            orders.start_date, orders.auto_expire_date,SUM(o.quantity)
+        SELECT m.patient_id, o.order_id, orders.encounter_id , m.visit_date,
+          d.drug_id, d.name, d.units, o.equivalent_daily_dose, orders.instructions,
+          orders.start_date, orders.auto_expire_date,SUM(o.quantity)
         FROM #{schema}.data_sanitizer_arv_medication m
         INNER JOIN #{schema}.drug_order o ON o.order_id = m.order_id
         INNER JOIN #{schema}.drug d ON d.drug_id = o.drug_inventory_id
@@ -856,36 +857,36 @@ module OpenmrsDataSanitizer
         INSERT INTO #{schema}.data_sanitizer_patient_outcomes(patient_id, visit_date, end_date, state, concept_id, outcome)
         SELECT p.patient_id,s.start_date, s.end_date, s.state, pws.concept_id, cn.name
         FROM #{schema}.patient_program p
-        INNER JOIN #{schema}.patient_state s ON p.patient_program_id = s.patient_program_id
-        AND p.program_id = 1 AND p.voided = 0 AND s.voided = 0 AND s.state IS NOT NULL
+        INNER JOIN #{schema}.patient_state s ON p.patient_program_id = s.patient_program_id AND p.program_id = 1 AND p.voided = 0 AND s.voided = 0 AND s.state IS NOT NULL
         INNER JOIN #{schema}.program_workflow_state pws ON s.state = pws.program_workflow_state_id AND pws.retired = 0 AND pws.terminal = 1
         INNER JOIN #{schema}.concept_name cn ON cn.concept_id = pws.concept_id AND cn.voided = 0 AND cn.name IS NOT NULL AND LENGTH(cn.name) > 0
         GROUP BY p.patient_id, DATE(s.start_date), DATE(p.date_enrolled)
-        ORDER BY DATE(s.start_date) DESC, DATE(p.date_enrolled) DESC;
       SQL
     end
 
     def insert_data_sanitizer_patient_who_stage
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_patient_who_stage(patient_id, visit_date, who_stage)
-        SELECT e.patient_id, DATE(t1.obs_datetime) visit_date, cn.name FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs t1 ON t1.encounter_id = e.encounter_id AND e.voided = 0
-        AND t1.voided = 0 AND e.encounter_type = 52 AND t1.concept_id = 7562
-        INNER JOIN #{schema}.concept_name cn ON cn.concept_id = t1.value_coded AND cn.voided = 0
-        WHERE LENGTH(cn.name) > 0
-        GROUP BY e.patient_id ORDER BY t1.obs_datetime DESC, t1.date_created DESC
+        SELECT e.patient_id, DATE(t1.obs_datetime) visit_date, cn.name
+        FROM #{schema}.encounter e
+        INNER JOIN #{schema}.obs t1 ON t1.encounter_id = e.encounter_id
+          AND t1.voided = 0 AND t1.concept_id = 7562
+        INNER JOIN #{schema}.concept_name cn ON cn.concept_id = t1.value_coded AND cn.voided = 0 AND LENGTH(cn.name) > 0
+        WHERE e.voided = 0 and e.encounter_type = 52 and e.program_id = 1
+        GROUP BY e.patient_id
       SQL
     end
 
     def insert_data_sanitizer_patient_reason_for_starting
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_patient_reason_for_starting(patient_id, visit_date, reason_for_starting_art)
-        SELECT e.patient_id, DATE(t1.obs_datetime) visit_date, cn.name FROM #{schema}.encounter e
-        INNER JOIN #{schema}.obs t1 ON t1.encounter_id = e.encounter_id AND e.voided = 0
-        AND t1.voided = 0 AND e.encounter_type = 52 AND t1.concept_id = 7563
-        INNER JOIN #{schema}.concept_name cn ON cn.concept_id = t1.value_coded AND cn.voided = 0
-        WHERE LENGTH(cn.name) > 0
-        GROUP BY e.patient_id ORDER BY t1.obs_datetime DESC, t1.date_created DESC;
+        SELECT e.patient_id, DATE(t1.obs_datetime) visit_date, cn.name
+        FROM #{schema}.encounter e
+        INNER JOIN #{schema}.obs t1 ON t1.encounter_id = e.encounter_id
+          AND t1.voided = 0 AND t1.concept_id = 7563
+        INNER JOIN #{schema}.concept_name cn ON cn.concept_id = t1.value_coded AND cn.voided = 0 AND LENGTH(cn.name) > 0
+        WHERE e.voided = 0 and e.encounter_type = 52 and e.program_id = 1
+        GROUP BY e.patient_id
       SQL
     end
 

data/app/services/openmrs_data_sanitizer/data_sanitizer.rb

@@ -13,17 +13,11 @@ module OpenmrsDataSanitizer
   # rubocop:disable Metrics/ClassLength
   # AWS DataSanitizer
   class DataSanitizer
-    attr_accessor :fhir_encounters, :fhir_observations, :regimen_combinations,
-                  :concept_names, :faker_names, :sites, :patients_json,
-                  :fhir_medication_dispense
+    attr_accessor :regimen_combinations, :concept_names, :faker_names, :sites
 
     def initialize(sites: [])
-      @fhir_encounters = Concurrent::Array.new
-      @fhir_observations = Concurrent::Array.new
-      @fhir_medication_dispense = Concurrent::Array.new
       @concept_names = Concurrent::Hash.new
       @faker_names = Concurrent::Hash.new
-      @patients_json = Concurrent::Array.new
       @regimen_combinations = {}
       @sites = sites
       load_regimen_combinations
@@ -63,13 +57,29 @@ module OpenmrsDataSanitizer
             end
             next unless loc
 
-            process_location(loc)
+            thread_variables # Set thread variables before processing
+            process_location(loc) # Process location
+            write_data_to_files # Write data to files
+            clear_thread_variables # Clear thread variables after processing
           end
         end
       end
 
       threads.each(&:join)
-      write_data_to_files
+    end
+
+    def thread_variables
+      Thread.current[:fhir_encounters] = Concurrent::Array.new
+      Thread.current[:fhir_observations] = Concurrent::Array.new
+      Thread.current[:fhir_medication_dispense] = Concurrent::Array.new
+      Thread.current[:patients_json] = Concurrent::Array.new
+    end
+
+    def clear_thread_variables
+      Thread.current[:fhir_encounters] = nil
+      Thread.current[:fhir_observations] = nil
+      Thread.current[:fhir_medication_dispense] = nil
+      Thread.current[:patients_json] = nil
     end
 
     def process_location(loc)
@@ -92,19 +102,18 @@ module OpenmrsDataSanitizer
 
     def process_patients(schema, site_name)
       patients = fetch_patients(schema:)
-      patients_ids = patients.map { |patient| patient['person_id'].to_i }
       patients.map { |patient| build_patient_json(patient, site_name) }
 
-      visits = fetch_visits(patients_ids:, schema:)
+      visits = fetch_visits(schema:)
       process_visits(visits:)
       process_medication(schema:)
     end
 
     def write_data_to_files
-      write_to_file(patients_json.to_json, 'patients')
-      write_to_file(fhir_encounters.to_json, 'visits')
-      write_to_file(fhir_observations.to_json, 'observations')
-      write_to_file(fhir_medication_dispense.to_json, 'medications')
+      write_to_file(Thread.current[:patients_json].to_json, 'patients')
+      write_to_file(Thread.current[:fhir_encounters].to_json, 'visits')
+      write_to_file(Thread.current[:fhir_observations].to_json, 'observations')
+      write_to_file(Thread.current[:fhir_medication_dispense].to_json, 'medications')
     end
 
     # ===================================
@@ -141,15 +150,7 @@ module OpenmrsDataSanitizer
 
     def fetch_patients(schema:)
       ActiveRecord::Base.connection.select_all <<~SQL
-        SELECT
-          p.person_id, given_name, middle_name, family_name, gender, birthdate,
-          phone.value phone_number, village.value village_name
-        FROM #{schema}.person p
-        INNER JOIN #{schema}.patient ON patient.patient_id = p.person_id AND p.voided = 0
-        INNER JOIN #{schema}.patient_program pp ON pp.patient_id = p.person_id AND pp.voided = 0 AND pp.program_id = 1
-        LEFT JOIN #{schema}.person_name n ON n.person_id = p.person_id AND n.voided = 0
-        LEFT JOIN #{schema}.person_attribute phone ON phone.person_id = p.person_id AND phone.voided = 0 AND phone.person_attribute_type_id = 12
-        LEFT JOIN #{schema}.person_attribute village ON village.person_id = p.person_id AND village.voided = 0 AND village.person_attribute_type_id = 19;
+        SELECT * FROM #{schema}.data_sanitizer_patients;
       SQL
     end
 
@@ -162,7 +163,7 @@ module OpenmrsDataSanitizer
       given_name = get_or_generate_faker_value(patient['given_name'], Faker::Name, :first_name)
       middle_name = get_or_generate_faker_value(patient['middle_name'], Faker::Name, :middle_name)
 
-      patients_json << FHIR::Patient.new(
+      Thread.current[:patients_json] << FHIR::Patient.new(
         id: patient_id,
         identifier: [{ system: "#{site_name}_#{patient_id}", value: patient_id }],
         name: [{ use: 'official', family: family_name, given: [given_name, middle_name] }],
@@ -172,19 +173,29 @@ module OpenmrsDataSanitizer
         telecom: [{ system: 'phone', value: cell_number, use: 'mobile' }]
       )
     end
-    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
     def fetch_gender(gender:)
-      gender.match(/f/i) ? 'female' : 'male'
+      case gender.strip.downcase
+      when 'f'
+        'female'
+      when 'm'
+        'male'
+      when 'u'
+        'unknown'
+      else
+        'unknown'
+      end
     rescue StandardError => e
       Rails.logger.info("Error fetching gender for patient: #{e.message}")
       'unknown'
     end
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
-    def fetch_visits(patients_ids:, schema:)
+    def fetch_visits(schema:)
       ActiveRecord::Base.connection.select_all <<~SQL
-        SELECT * FROM #{schema}.data_sanitizer_visits
-        WHERE patient_id IN(#{patients_ids.join(',')})
+        SELECT dsv.*
+        FROM #{schema}.data_sanitizer_visits dsv
+        INNER JOIN #{schema}.data_sanitizer_patients dsp ON dsp.patient_id = dsv.patient_id
       SQL
     end
 
@@ -202,7 +213,7 @@ module OpenmrsDataSanitizer
 
     def aws_fhir_encounter(visit:, regimen:)
       patient_id = visit['patient_id'].to_i
-      fhir_encounters << create_fhir_encounter(patient_id, visit['visit_date'])
+      Thread.current[:fhir_encounters] << create_fhir_encounter(patient_id, visit['visit_date'])
 
       observations = process_indicators(visit, regimen)
       create_fhir_observations(observations, patient_id, visit['visit_date'])
@@ -269,7 +280,7 @@ module OpenmrsDataSanitizer
       observations.each do |obs|
         next if obs.blank? || obs[:code].blank? || obs[:value].blank?
 
-        fhir_observations << FHIR::Observation.new(
+        Thread.current[:fhir_observations] << FHIR::Observation.new(
           status: 'final',
           code: {
             coding: [
@@ -284,20 +295,20 @@ module OpenmrsDataSanitizer
             reference: "Patient/#{patient_id}"
           },
           encounter: {
-            reference: "Encounter/#{fhir_encounters.last.id}"
+            reference: "Encounter/#{Thread.current[:fhir_encounters].last.id}"
           },
           effectiveDateTime: visit_date
         )
 
         if obs[:unit]
-          fhir_observations.last.valueQuantity = {
+          Thread.current[:fhir_observations].last.valueQuantity = {
             value: obs[:value],
             unit: obs[:unit],
             system: 'EGPAF Malawi EMR',
             code: obs[:code]
           }
         else
-          fhir_observations.last.valueCodeableConcept = {
+          Thread.current[:fhir_observations].last.valueCodeableConcept = {
             coding: [
               {
                 system: 'EGPAF Malawi EMR',
@@ -332,16 +343,14 @@ module OpenmrsDataSanitizer
 
     # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/BlockLength
     def create_fhir_medications(orders:)
-      fhir_medication_dispense << (orders || []).map do |order|
-        order_id = order['order_id'].to_i
-        encounter_id = order['encounter_id'].to_i
+      Thread.current[:fhir_medication_dispense] << (orders || []).map do |order|
         FHIR::MedicationDispense.new(
           identifier: {
-            value: order_id
+            value: order['order_id'].to_i
           },
           status: 'completed',
           patient: {
-            reference: encounter_id
+            reference: order['encounter_id'].to_i
           },
           medication: {
             code: {
@@ -388,7 +397,7 @@ module OpenmrsDataSanitizer
     def get_or_generate_faker_value(attribute, faker_module, faker_method)
       return '' if attribute.blank?
 
-      @faker_names[attribute] ||= faker_module.send(faker_method)
+      faker_names[attribute] ||= faker_module.send(faker_method)
     end
 
     def get_concept_name(name)
@@ -411,9 +420,14 @@ module OpenmrsDataSanitizer
 
       # Use the mutex to handle concurrent writes
       FILE_MUTEX.synchronize do
+        if File.exist?(file_path)
+          system("sed -i '$s/\([][^{},]\)$/,/g' #{file_path}")
+          json_file.sub!('[', '')
+        end
+
         File.open(file_path, 'a') do |out_file|
           out_file.puts(json_file)
-          puts "Created file: #{file_path}"
+          Rails.logger.info "Appended to file: #{file_path}"
         end
       end
     end

data/lib/openmrs_data_sanitizer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OpenmrsDataSanitizer
-  VERSION = '1.0.3'
+  VERSION = '1.0.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: openmrs_data_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Mwatha
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2024-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby