openmrs_data_sanitizer 1.0.6 → 1.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45352f6a9ff782926a46e2e93973f3a7873b8316bf482481d09c1543de7b958c
-  data.tar.gz: ef10ff0800b3895cd22de64c0171255d3870888b15fb302d41f4c0fff5acc89d
+  metadata.gz: 40a55f5fb47f9307da0c2e0a5905f5554dee6688c1f4abf09889c4a7852fca95
+  data.tar.gz: 18ebfa45e8aefdd2fd3bf0cab02d0187ba9d3ac16e4f37e50a9de982cacd4e6d
 SHA512:
-  metadata.gz: a931566411fae2f5cf78b5e59757a9ae7c2610c6916d67610623a8906cd1865c94b3c2bca2adda5ca9737e601a4d983e1a36349ad305ce7beb4a186905d22e33
-  data.tar.gz: 9748ee31837097d5f9d4c1210f79b569f678dc78feb7a841c7cac65b5a9332e03cb12f2805727d9b89c7c273f2f38d464253edc9e6fd3785029f1cc433d41fa4
+  metadata.gz: 6db161bf332f91467f74e8f489cc536d06aac21689dc8114e296d70fbb1851d25a58face6b05141634d74806bbd9ea03a17d509ba87e8605e22a94795c8902fd
+  data.tar.gz: 9b0dfacd32c8bbbf5d9154bcf1402878e279bbf756ee37300d179e8d456d43b589ba857af87156619372679c798c713eae2505b42d671ed185db8c9b4884a1f1

data/app/services/openmrs_data_sanitizer/data_aggregator.rb

@@ -681,7 +681,7 @@ module OpenmrsDataSanitizer
         FROM #{schema}.encounter e
         INNER JOIN #{schema}.obs ON e.encounter_id = obs.encounter_id AND obs.voided = 0 AND obs.concept_id = 5090 -- Height concept_id
         WHERE e.voided = 0 and e.encounter_type = 6 and e.program_id = 1
-        GROUP BY obs.person_id, DATE(obs.obs_datetime);
+        GROUP BY e.patient_id, DATE(obs.obs_datetime);
       SQL
     end
 
@@ -694,7 +694,7 @@ module OpenmrsDataSanitizer
           AND obs.voided = 0
           AND obs.concept_id = 5089
         WHERE e.voided = 0 and e.encounter_type = 6 and e.program_id = 1
-        GROUP BY obs.person_id, DATE(obs.obs_datetime);
+        GROUP BY e.patient_id, DATE(obs.obs_datetime);
       SQL
     end
 
@@ -808,37 +808,35 @@ module OpenmrsDataSanitizer
           AND obs.voided = 0
           AND obs.concept_id = 5096
         WHERE e.voided = 0 and e.encounter_type = 7 and e.program_id = 1
-        GROUP BY obs.person_id, DATE(obs.obs_datetime);
+        GROUP BY e.patient_id, DATE(obs.obs_datetime);
       SQL
     end
 
     def insert_data_sanitizer_arv_medication
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_arv_medication (patient_id, order_id, visit_date, drug_runout_date, num_unique_arvs, unique_arvs_given, drug_names)
-        SELECT e.patient_id, o.order_id, DATE(o.start_date) AS visit_date, DATE(MIN(o.auto_expire_date)) AS drug_runout_date,
+        SELECT o.patient_id, o.order_id, DATE(o.start_date) AS visit_date, DATE(MIN(o.auto_expire_date)) AS drug_runout_date,
           COUNT(DISTINCT d.drug_id) AS num_unique_arvs,GROUP_CONCAT(DISTINCT d.drug_id SEPARATOR ', ') AS unique_arvs_given,
           GROUP_CONCAT(DISTINCT d.name SEPARATOR ', ') AS drug_names
         FROM #{schema}.orders o
         INNER JOIN #{schema}.drug_order da ON da.order_id = o.order_id AND o.voided = 0 AND da.quantity > 0
         INNER JOIN #{schema}.drug d ON d.drug_id = da.drug_inventory_id
-        INNER JOIN #{schema}.obs ON obs.order_id = o.order_id AND obs.voided = 0
-        INNER JOIN #{schema}.encounter e ON e.program_id = 1 AND e.encounter_id = obs.encounter_id AND e.encounter_type = 54
+        INNER JOIN #{schema}.encounter e ON e.program_id = 1 AND e.encounter_id = o.encounter_id AND e.encounter_type = 54
         INNER JOIN #{schema}.arv_drug ON arv_drug.drug_id = d.drug_id
-        GROUP BY obs.person_id, o.order_id, DATE(o.start_date);
+        GROUP BY o.patient_id, DATE(o.start_date);
       SQL
     end
 
     def insert_data_sanitizer_arv_orders
       ActiveRecord::Base.connection.execute <<~SQL
         INSERT INTO #{schema}.data_sanitizer_arv_orders(patient_id, order_id, encounter_id, visit_date, drug_id, drug_name, units, equivalent_daily_dose, instructions, start_date, runout_date, quantity)
-        SELECT m.patient_id, o.order_id, orders.encounter_id , m.visit_date,
-          d.drug_id, d.name, d.units, o.equivalent_daily_dose, orders.instructions,
-          orders.start_date, orders.auto_expire_date,SUM(o.quantity)
-        FROM #{schema}.data_sanitizer_arv_medication m
-        INNER JOIN #{schema}.drug_order o ON o.order_id = m.order_id
-        INNER JOIN #{schema}.drug d ON d.drug_id = o.drug_inventory_id
-        INNER JOIN #{schema}.orders ON orders.order_id = o.order_id
-        GROUP BY d.drug_id, o.order_id, m.visit_date;
+        SELECT e.patient_id, o.order_id, o.encounter_id, DATE(o.start_date) AS visit_date, da.drug_inventory_id AS drug_id, d.name AS drug_name, d.units, da.equivalent_daily_dose, o.instructions, DATE(o.start_date), DATE(o.auto_expire_date), da.quantity
+        FROM #{schema}.orders o
+        INNER JOIN #{schema}.encounter e ON e.encounter_id = o.encounter_id AND e.program_id = 1 AND e.voided = 0
+        INNER JOIN #{schema}.drug_order da ON da.order_id = o.order_id AND da.quantity > 0
+        INNER JOIN #{schema}.drug d ON d.drug_id = da.drug_inventory_id
+        INNER JOIN #{schema}.arv_drug ad ON ad.drug_id = d.drug_id
+        WHERE o.voided = 0
       SQL
     end
 

data/app/services/openmrs_data_sanitizer/data_sanitizer.rb

@@ -13,13 +13,14 @@ module OpenmrsDataSanitizer
   # rubocop:disable Metrics/ClassLength
   # AWS DataSanitizer
   class DataSanitizer
-    attr_accessor :regimen_combinations, :concept_names, :faker_names, :sites
+    attr_accessor :regimen_combinations, :concept_names, :faker_names, :sites, :skip
 
-    def initialize(sites: [])
+    def initialize(sites: [], skip: true)
       @concept_names = Concurrent::Hash.new
       @faker_names = Concurrent::Hash.new
       @regimen_combinations = {}
       @sites = sites
+      @skip = skip
       load_regimen_combinations
     end
 
@@ -47,7 +48,7 @@ module OpenmrsDataSanitizer
       queue = Queue.new
       sites.each { |loc| queue << loc }
 
-      threads = Array.new(10) do
+      threads = Array.new(60) do
         Thread.new do
           until queue.empty?
             loc = begin
@@ -57,29 +58,24 @@ module OpenmrsDataSanitizer
             end
             next unless loc
 
-            thread_variables # Set thread variables before processing
             process_location(loc) # Process location
-            write_data_to_files # Write data to files
-            clear_thread_variables # Clear thread variables after processing
           end
         end
       end
 
       threads.each(&:join)
+      close_files
     end
 
-    def thread_variables
-      Thread.current[:fhir_encounters] = Concurrent::Array.new
-      Thread.current[:fhir_observations] = Concurrent::Array.new
-      Thread.current[:fhir_medication_dispense] = Concurrent::Array.new
-      Thread.current[:patients_json] = Concurrent::Array.new
-    end
-
-    def clear_thread_variables
-      Thread.current[:fhir_encounters] = nil
-      Thread.current[:fhir_observations] = nil
-      Thread.current[:fhir_medication_dispense] = nil
-      Thread.current[:patients_json] = nil
+    def close_files
+      dir_path = Rails.root.join('log/openmrs_data_sanitizer')
+      file_path = dir_path.join('*.json')
+      Dir[file_path].each do |file|
+        # Remove trailing comma and append closing bracket
+        system("sed -i '$s/,$/]/' #{file}")
+        # Log the file closure
+        Rails.logger.info "Closed file: #{file}"
+      end
     end
 
     def process_location(loc)
@@ -90,7 +86,7 @@ module OpenmrsDataSanitizer
         site_name = site['facility']
         schema = site['schema']
 
-        OpenmrsDataSanitizer::DataAggregator.new(schema:).aggregate
+        OpenmrsDataSanitizer::DataAggregator.new(schema:).aggregate unless skip
 
         process_patients(schema, site_name)
       rescue StandardError => e
@@ -109,13 +105,6 @@ module OpenmrsDataSanitizer
       process_medication(schema:)
     end
 
-    def write_data_to_files
-      write_to_file(Thread.current[:patients_json].to_json, 'patients') unless Thread.current[:patients_json].blank?
-      write_to_file(Thread.current[:fhir_encounters].to_json, 'visits') unless Thread.current[:fhir_encounters].blank?
-      write_to_file(Thread.current[:fhir_observations].to_json, 'observations') unless Thread.current[:fhir_observations].blank?
-      write_to_file(Thread.current[:fhir_medication_dispense].to_json, 'medications') unless Thread.current[:fhir_medication_dispense].blank?
-    end
-
     # ===================================
     #  REGION: REGIMEN COMBINATIONS
     # ===================================
@@ -156,14 +145,14 @@ module OpenmrsDataSanitizer
 
     # rubocop:disable Metrics/AbcSize
     def build_patient_json(patient, site_name)
-      patient_id = patient['person_id'].to_i
-      village_name = get_or_generate_faker_value(patient['village_name'], Faker::Address, :community)
-      cell_number = get_or_generate_faker_value(patient['phone_number'], Faker::PhoneNumber, :phone_number)
-      family_name = get_or_generate_faker_value(patient['family_name'], Faker::Name, :last_name)
-      given_name = get_or_generate_faker_value(patient['given_name'], Faker::Name, :first_name)
-      middle_name = get_or_generate_faker_value(patient['middle_name'], Faker::Name, :middle_name)
-
-      Thread.current[:patients_json] << FHIR::Patient.new(
+      patient_id = patient['patient_id'].to_i
+      village_name = get_or_generate_faker_value(patient['village_name']&.titlecase, Faker::Address, :community)
+      cell_number = get_or_generate_faker_value(patient['phone_number']&.titlecase, Faker::PhoneNumber, :phone_number)
+      family_name = get_or_generate_faker_value(patient['family_name']&.titlecase, Faker::Name, :last_name)
+      given_name = get_or_generate_faker_value(patient['given_name']&.titlecase, Faker::Name, :first_name)
+      middle_name = get_or_generate_faker_value(patient['middle_name']&.titlecase, Faker::Name, :middle_name)
+
+      fhir_patient = FHIR::Patient.new(
         id: patient_id,
         identifier: [{ system: "#{site_name}_#{patient_id}", value: patient_id }],
         name: [{ use: 'official', family: family_name, given: [given_name, middle_name] }],
@@ -172,6 +161,7 @@ module OpenmrsDataSanitizer
         address: [{ use: 'home', village: village_name }],
         telecom: [{ system: 'phone', value: cell_number, use: 'mobile' }]
       )
+      write_to_file(fhir_patient.to_json, 'patients') unless fhir_patient.blank?
     end
 
     def fetch_gender(gender:)
@@ -213,10 +203,11 @@ module OpenmrsDataSanitizer
 
     def aws_fhir_encounter(visit:, regimen:)
       patient_id = visit['patient_id'].to_i
-      Thread.current[:fhir_encounters] << create_fhir_encounter(patient_id, visit['visit_date'])
+      fhir_encounter = create_fhir_encounter(patient_id, visit['visit_date'])
+      write_to_file(fhir_encounter.to_json, 'visits') unless fhir_encounter.blank?
 
       observations = process_indicators(visit, regimen)
-      create_fhir_observations(observations, patient_id, visit['visit_date'])
+      create_fhir_observations(observations, patient_id, visit['visit_date'], fhir_encounter.id)
     end
 
     # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
@@ -276,11 +267,11 @@ module OpenmrsDataSanitizer
     end
 
     # rubocop:disable Metrics/PerceivedComplexity
-    def create_fhir_observations(observations, patient_id, visit_date)
+    def create_fhir_observations(observations, patient_id, visit_date, encounter_id)
       observations.each do |obs|
         next if obs.blank? || obs[:code].blank? || obs[:value].blank?
 
-        Thread.current[:fhir_observations] << FHIR::Observation.new(
+        fhir_obs = FHIR::Observation.new(
           status: 'final',
           code: {
             coding: [
@@ -295,20 +286,19 @@ module OpenmrsDataSanitizer
             reference: "Patient/#{patient_id}"
           },
           encounter: {
-            reference: "Encounter/#{Thread.current[:fhir_encounters].last.id}"
+            reference: "Encounter/#{encounter_id}"
           },
           effectiveDateTime: visit_date
         )
-
         if obs[:unit]
-          Thread.current[:fhir_observations].last.valueQuantity = {
+          fhir_obs.valueQuantity = {
             value: obs[:value],
             unit: obs[:unit],
             system: 'EGPAF Malawi EMR',
             code: obs[:code]
           }
         else
-          Thread.current[:fhir_observations].last.valueCodeableConcept = {
+          fhir_obs.valueCodeableConcept = {
             coding: [
               {
                 system: 'EGPAF Malawi EMR',
@@ -322,6 +312,7 @@ module OpenmrsDataSanitizer
             ]
           }
         end
+        write_to_file(fhir_obs.to_json, 'observations') unless fhir_obs.blank?
       end
     end
     # rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength, Metrics/PerceivedComplexity
@@ -343,8 +334,8 @@ module OpenmrsDataSanitizer
 
     # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/BlockLength
     def create_fhir_medications(orders:)
-      Thread.current[:fhir_medication_dispense] << (orders || []).map do |order|
-        FHIR::MedicationDispense.new(
+      (orders || []).each do |order|
+        fhir_med = FHIR::MedicationDispense.new(
           identifier: {
             value: order['order_id'].to_i
           },
@@ -386,6 +377,7 @@ module OpenmrsDataSanitizer
             }
           }
         )
+        write_to_file(fhir_med.to_json, 'medications') unless fhir_med.blank?
       end
     end
     # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/BlockLength
@@ -420,14 +412,15 @@ module OpenmrsDataSanitizer
 
       # Use the mutex to handle concurrent writes
       FILE_MUTEX.synchronize do
-        if File.exist?(file_path)
-          system("sed -i '$s/\([][^{},]\)$/,/g' #{file_path}")
-          json_file.sub!('[', '')
-          Rails.logger.info "Removed last bracket from file: #{file_path}"
+        if !File.exist?(file_path)
+          File.open(file_path, 'w') do |out_file|
+            out_file.puts('[')
+            Rails.logger.info "Created file: #{file_path}"
+          end
         end
 
         File.open(file_path, 'a') do |out_file|
-          out_file.puts(json_file)
+          out_file.puts("#{json_file},")
           Rails.logger.info "Appended to file: #{file_path}"
         end
       end

data/lib/openmrs_data_sanitizer/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OpenmrsDataSanitizer
-  VERSION = '1.0.6'
+  VERSION = '1.0.8'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: openmrs_data_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.8
 platform: ruby
 authors:
 - Mwatha
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-03 00:00:00.000000000 Z
+date: 2024-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.17
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: This gem will mask all patient identifyable data before exporting to AWS