hlsv 1.0.0 → 2.0.0

data/lib/hlsv/sdtm_validation.rb

@@ -0,0 +1,390 @@
+require 'yaml'
+require 'nokogiri'
+require 'fileutils'
+require 'csv'
+
+require_relative 'xpt'
+
+require_relative "html2word"
+
+require_relative 'sdtm_validation/dataset'
+require_relative 'sdtm_validation/define'
+require_relative 'sdtm_validation/report'
+
+module Hlsv
+class SdtmValidation
+
+  attr_reader :web_mode
+  attr_reader :verbose
+
+  attr_reader :config
+
+  attr_reader :define
+  attr_reader :datasets
+  attr_reader :ds_path
+
+  def initialize(config_file, web_mode: false, verbose: false)
+    @web_mode = web_mode
+    @verbose = verbose
+
+    # load the config
+    @config = load_config(config_file)
+
+    # load define
+    @define = Define.new(@config['define_path'])
+
+    # load data
+    # get all xpt files in the directory present in the config file (data_directory parameter)
+    @ds_path = @config['data_directory'].gsub('\\', '/')
+
+    unless File.directory? @ds_path
+      raise Hlsv::Error, "⚠ dataset directory is not a directory, analysis not possible: #{@ds_path}"
+    end
+
+    # present of excluded datasets
+    puts "Excluded dataset: #{excluded_dataset.join(', ')}" unless excluded_dataset.empty?
+
+    @datasets = Dir["#{@ds_path}/*"]
+      .select { |f| File.extname(f) == '.xpt' }
+      .reject { |f| excluded_dataset.include? File.basename(f, '.xpt').upcase }
+      .to_h { |ds| [File.basename(ds, '.xpt').upcase, Dataset.from_xpt(ds)] }
+
+    if @datasets.empty?
+      raise Hlsv::Error, "⚠ no xpt files detected, analysis not possible"
+    end
+
+    # refresh the output directory
+    prepare_output_dir
+
+  end
+
+  def validate
+    ascii_search
+    check_define_key if @define.status
+    check_data_key
+  end
+
+  # check config key in the data
+  def check_data_key
+
+    puts "\n=== data check"
+
+    @datasets.each do |ds, dataset|
+      puts ds
+
+      # get candidate keys
+      candidates = dataset.candidates_keys(ds, @config)
+
+      # get the dataset records
+      records = @datasets[ds].records
+
+      # no records present
+      if records.empty?
+        puts '- no records present'
+        next
+      end
+
+      # search the minimal combination of variables to avoid duplicates
+      candidates.each_with_index  do |candidate, i|
+
+        if candidates.size == 1
+          idx = i;
+        else
+          idx = i+1
+        end
+
+        duplicate_records = look_keys(ds, candidate, records)
+
+        unless duplicate_records.empty?
+          dup_file = write_csv(ds, duplicate_records, :data, idx)
+
+          @datasets[ds].data_duplicate_file << dup_file
+        end
+      end
+    end
+  end
+
+  private def look_keys(ds, keys_candidates, records)
+    # initialize the number of duplicate with the records of records present in the dataset
+    prev_records_max = records.size
+    prev_nb_sub = records.size
+    # initialize the useless variables as an empty array
+    useless_vars = []
+
+    # display the variables check during the search
+    puts "- the key search is based on the following variables in the given order: #{keys_candidates.join(', ')}"
+
+    # loop on all candidate keys variables
+    keys_candidates.size.times do |i|
+
+      # tested key
+      keys_vars = keys_candidates[..i]
+      # remove useless variables
+      useless_vars.each do |v|
+        keys_vars.delete(v)
+      end
+      puts "  - The key tried is: #{keys_vars.join(', ')}" if @verbose
+
+      # array of values of the variables
+      keys_records = records.group_by do |p|
+        keys_vars.map { |var| p[var] }
+      end
+
+      # get the maximum of records by key
+      records_max = 0
+      nb_sub = 0
+      keys_records.each do |values, list|
+        records_max = list.size if list.size > records_max
+        if list.size > 1
+          nb_sub += 1
+        end
+      end
+
+      # break if no duplicate
+      if records_max == 1
+        puts "- the minimal valid key have been found: #{keys_vars.join(', ')}"
+        @datasets[ds].minimal_key << keys_vars
+        @datasets[ds].minimal_key_status << true
+        return []
+      end
+
+      # duplicate still present
+      puts "  - failure: #{nb_sub} subjects have a maximum #{records_max} records for #{keys_vars.join(', ')}" if @verbose
+      # if the number of current duplicate is superior or equal to the previous
+      # number of duplicate then the variable is not useful for the key search and
+      # the previous number of maximum duplicate stay as is
+      # the current variable will be remove for the next search
+      if prev_records_max <= records_max && prev_nb_sub <= nb_sub
+        puts "  - #{keys_vars.last} not useful: The number of subjects and the maximum number of duplicates are the same as for the previous tested key." if @verbose
+        useless_vars << keys_vars.last
+      end
+
+      # current duplicate number become the current number of duplicate
+      prev_records_max = records_max
+      prev_nb_sub = nb_sub
+
+      # duplicate if no more variables
+      if i == (keys_candidates.size-1)
+        # duplicates records with the last valid key
+        last_valid_keys = keys_candidates.reject { |kc| useless_vars.include? kc }
+        puts "- The minimal key has not been found. The last key that was tested is: #{last_valid_keys.join(', ')}"
+        @datasets[ds].minimal_key << last_valid_keys
+        @datasets[ds].minimal_key_status << false
+
+        # compile all duplicates with the last valid keys
+        duplicate_records = records.group_by do |p|
+          last_valid_keys.map { |var| p[var] }
+        end
+        @datasets[ds].data_duplicate_records << duplicate_records
+
+        return duplicate_records
+      end
+    end
+
+  end
+
+  # check define key validity
+  def check_define_key
+
+    puts "\n=== define check"
+
+    # get all datasets: present in data & in define
+    ds_list = (@datasets.values.map(&:name) + @define.datasets).uniq
+
+    ds_list.each do |ds|
+
+      # no check if excluded dataset
+      if excluded_dataset.include? ds
+        @define.keys.delete(ds)
+        next
+      end
+
+      # get the define keys
+      define_key = @define.define_key_for(ds)
+
+      # no define key
+      if define_key.nil?
+        puts "#{ds}: no define key"
+        next
+      end
+      # transform key in symbol
+      define_key.map!(&:to_sym)
+
+      # add into dataset
+      if @datasets.keys.include? ds
+        @datasets[ds].define_keys = define_key
+      else
+        puts "#{ds}: present in define.xml but not in dataset"
+        @datasets[ds] = Dataset.new(ds, nil, [])
+      end
+
+      # get the records
+      records = @datasets[ds].records
+
+      # no records
+      if records.empty?
+        puts "#{ds}: no records for define datasets"
+        next
+      end
+
+      # define key & records are present
+      keys_records = records.group_by do |record|
+        define_key.map { |var| record[var] }
+      end
+
+      records_max = keys_records.map { |_, list| list.size }.max || 0
+
+      if records_max == 1
+        puts "#{ds}: define key is valid"
+        @datasets[ds].define_status = true
+      else
+        puts "#{ds}: define key is invalid"
+        @datasets[ds].define_status = false
+        @datasets[ds].define_duplicate_records = keys_records
+
+        dup_file = write_csv(ds, keys_records, :define, 0)
+        @datasets[ds].define_duplicate_file = dup_file
+      end
+    end
+  end
+
+  # search non-ASCII character
+  def ascii_search
+
+    puts "\n=== Non-ASCII Characters Search"
+
+    @datasets.values.each do |ds|
+      records = ds.records
+      # no records no test
+      next if records.empty?
+      # initialize list of exception
+      non_ascii_list = []
+      # read all characters
+      records.each do |record|
+        record.each do |var, value|
+          next if value.nil? || value.is_a?(Integer) || value.is_a?(Float)
+          if value.match(/[^[:ascii:]]/)
+            non_ascii_list << "#{var}: #{value.inspect}"
+          end
+        end
+      end
+
+      ds.non_ascii_values = non_ascii_list
+
+      if non_ascii_list.empty?
+        puts "#{ds.name}: non-ASCII absent"
+      else
+        puts "#{ds.name}: non-ASCII present", non_ascii_list.map { |v| "- #{v}" }
+      end
+    end
+  end
+
+  # report creation
+  def report
+    @report ||= Report.new(study_name, @ds_path, @datasets, @define, excluded_dataset, web_mode: @web_mode)
+  end
+
+  ####
+  # load YAML file
+  # the validation on the config is done by the web app
+  # return hash: parameter => value
+  def load_config(file)
+    YAML.load_file(file)
+  rescue Psych::SyntaxError => e
+    raise Hlsv::Error, "❌ Invalid configuration\n\n> #{e.message}"
+  rescue Errno::ENOENT
+    raise Hlsv::Error, "❌ File not found\n\n> `#{file}` does not exist"
+  end
+
+  ###
+  # display
+  #
+  # create a CVS with all duplicates
+  def write_csv(ds, duplicate_records, type, index)
+
+    if index == 0 then
+      display_out = "#{dir_out}/#{type.to_s}_#{ds}.csv"
+    else
+      display_out = "#{dir_out}/#{type.to_s}_#{index}_#{ds}.csv"
+    end
+
+    # TODO: hum ça marche ouais mais non, faut pas laisser comme ça
+    header = duplicate_records.first.last.first.keys
+    header = header.insert(0, 'No')
+
+    duplicate = duplicate_records.reject { |key, list| list.size == 1 }
+
+    CSV.open(display_out, "w") do |csv|
+      # header
+      csv << header
+      # body
+      duplicate.each do |key, list|
+        idx = duplicate.keys.index(key) + 1
+        list.each do |l|
+          row = l.values.insert(0, idx)
+          csv << row
+        end
+      end
+    end
+
+    display_out
+  end
+
+  ###
+  # Helpers
+
+  # study name
+  def study_name
+    @config['study_name']
+  end
+
+  # excluded ds
+  def excluded_dataset
+    @excluded_dataset ||= begin
+      exclus = @config['excluded_ds']&.gsub(/[,;]/, ' ')
+      if exclus.nil?
+        []
+      else
+        exclus.split.map(&:upcase)
+      end
+    end
+  end
+
+  # output directory
+  def prepare_output_dir
+    dir = "#{study_name}/#{config['output_directory']}"
+
+    # no value, current repository
+    if dir.nil? || dir.strip.empty?
+      dir = '.'
+
+    # repository present, delete all csv file
+    elsif Dir.exist?(dir)
+      Dir.glob("#{dir}/**/*.csv").each { |f| File.delete(f) }
+
+    # no repository, creation
+    else
+      FileUtils.mkdir_p(dir)
+    end
+
+    dir
+  end
+
+  def dir_out
+    dir = "#{study_name}/#{config['output_directory']}"
+    dir = '.' if dir.nil? || dir.strip.empty?
+    dir
+  end
+
+
+end
+
+# sv = SdtmValidation.new('../../config.yaml')
+# sv.validate
+# sv.report.html
+# sv.report.docx
+# sv.report.excel_export('SV')
+# sv.report.excel_export('DA')
+# sv.report.excel_export('DM')
+# sv.report.excel_export('DC')
+end

data/lib/hlsv/url_helper.rb

@@ -0,0 +1,28 @@
+# lib/hlsv/url_helper.rb
+module Hlsv
+  module UrlHelper
+    # Builds an encoded query string from a parameter hash.
+    # nil values are ignored (Useful for the optional `last_valid_key` option, etc.)
+    def build_query(params)
+      params.compact.map { |k, v| "#{k}=#{::URI.encode_www_form_component(v.to_s)}" }.join('&')
+    end
+
+    def csv_view_url(file:, study:, dataset:, last_valid_key: nil)
+      "/csv_view?#{build_query(
+        file: file,
+        study: study,
+        dataset: dataset,
+        last_valid_key: last_valid_key&.join(',')
+      )}"
+    end
+
+    def export_csv_excel_url(file:, study:, dataset:, last_valid_key: nil)
+      "/export_csv_excel?#{build_query(
+        file: file,
+        study: study,
+        dataset: dataset,
+        last_valid_key: last_valid_key&.join(',')
+      )}"
+    end
+  end
+end

data/lib/hlsv/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hlsv
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end