hlsv 1.0.0

data/lib/hlsv/find_keys.rb

@@ -0,0 +1,979 @@
+# Copyright (c) 2026 AdClin
+# Licensed under the GNU Affero General Public License v3.0 or later.
+# See the LICENSE file for details.
+
+require 'yaml'
+require 'nokogiri'
+require 'csv'
+require 'fileutils'
+require 'pathname'
+require 'erb'
+
+require 'spreadsheet'
+require 'rubyXL'
+require 'rubyXL/convenience_methods/cell'
+require 'rubyXL/convenience_methods/color'
+require 'rubyXL/convenience_methods/font'
+require 'rubyXL/convenience_methods/workbook'
+require 'rubyXL/convenience_methods/worksheet'
+
+
+require_relative 'xpt'
+require_relative 'html2word'
+module Hlsv
+class FindKeys
+
+  attr_reader :verbose
+  attr_reader :config
+  attr_reader :ds_list
+  attr_reader :study_name
+  attr_reader :web_mode
+
+  attr_accessor :report
+  attr_accessor :ds_with_issue
+
+  def initialize(config_file, verbose: false, web_mode: false)
+
+    @verbose = verbose
+    @web_mode= web_mode
+    @ds_with_issue = {
+      ascii: [],
+      define: [],
+      data: [],
+    }
+
+    report_init = {
+      config_name: nil,
+      data_information: {},
+      define_information: {},
+    }
+
+    # load the config file
+    # read YAML, csv, xls, xlsx only
+    @config = load_config(config_file)
+    if @web_mode
+      report_init[:config_name] = 'config.yaml'
+    else
+      report_init[:config_name] = config_file
+    end
+
+    # save the study name
+    @study_name = config['study_name']
+    report_name = "#{@study_name}/#{File.basename(@study_name)}_high_level_check.html"
+    @report = Report.new(report_name, @study_name, outname_ext)
+
+    # get all xpt files in the directory present in the config file (data_directory parameter)
+    dir = config['data_directory'].gsub('\\', '/')
+    @ds_list = Dir["#{dir}/*"].select { |f| File.extname(f) == '.xpt' }
+
+    report_init[:data_information] = {
+      directory_name: dir,
+      file_number: @ds_list.size,
+    }
+
+    # load define with Nokogiri gem
+    # get define keys
+    # return {dataset => array of variables}
+    @define_keys, define_report = load_define()
+    report_init[:define_information] = define_report
+
+    @report.add_init(report_init)
+
+    # remove all previous csv, in case of rerun
+    prepare_output_dir
+  end
+
+  ###
+  # Check some high level information
+  # Restriction: the dataset export is load only one time to avoid some performance issue
+  # Process:
+  # - Load the dataset records
+  # - Check the presence of non-ASCII character in data
+  # - Check the validity of the key that is referenced in define.xml.
+  # - Search the minimal key with the variables list present in the config file
+  def high_level_check
+
+    @ds_list.each do |dsf|
+      # store the dataset name
+      ds = File.basename(dsf).split('.')[0].upcase
+
+      # shortcut if dataset in the excluded list
+      next if config['excluded_ds'].include? ds
+
+      # load all records
+      # only XPT
+      # TODO json
+      records = load_dsfile(dsf)
+
+      # Initialiser la structure du rapport pour ce dataset
+      ds_report = {
+        record_count: records&.size,
+        candidates_type: nil,
+        ascii_check: nil,
+        define_key_check: nil,
+        data_key_check: nil,
+        verbose_details: []
+      }
+
+      # shortcut if no records in the file
+      if records.empty?
+        ds_report[:record_count] = 0
+        @report.add_dataset_report(ds, ds_report)
+        next
+      end
+
+      # search non-ASCII
+      ascii_issues, valid_ascii = ascii_search(records)
+      ds_report[:ascii_check] = {
+        valid: valid_ascii,
+        issues: ascii_issues
+      }
+
+      @ds_with_issue[:ascii] << ds unless valid_ascii
+
+      # check define keys
+      if @define_keys && @define_keys[ds]
+        define_key, valid_define, duplicate_file = key_check_define(ds, @define_keys[ds], records)
+        ds_report[:define_key_check] = {
+          valid: valid_define,
+          key: define_key,
+          duplicate_file: duplicate_file
+        }
+      elsif @define_keys == '-'
+        ds_report[:define_key_check] = {
+          valid: true,
+          absent: true,
+        }
+      else
+        ds_report[:define_key_check] = {
+          valid: false,
+          absent: true,
+        }
+      end
+
+      @ds_with_issue[:define] << ds unless valid_define
+
+      # check data keys
+      data_key_info, valid_data, candidates_type = key_check_data(ds, records)
+      ds_report[:data_key_check] = data_key_info
+      ds_report[:candidates_type] = candidates_type
+      @ds_with_issue[:data] << ds unless valid_data
+
+      @report.add_dataset_report(ds, ds_report)
+    end
+  end
+
+  # search key according to the config
+  def key_check_data(ds, records)
+    # Cas spéciaux pour SE et SV
+    if ds == 'SE'
+      keys_candidates_list = [
+        [:USUBJID, :EPOCH, :SUBJID],
+        [:USUBJID, :TAETORD, :SUBJID]
+      ]
+
+      results = keys_candidates_list.map do |keys_candidates|
+        result = look_keys(ds, keys_candidates, records)
+        if result[:valid]
+          { valid: true, key: result[:key], candidates: keys_candidates }
+        else
+          { valid: false, last_valid_key: result[:last_valid_key],
+            duplicate_file: result[:duplicate_file], candidates: keys_candidates }
+        end
+      end
+
+      # Retourner un array de résultats et valid si au moins une clé est valide
+      [results, results.any? { |r| r[:valid] }]
+
+    elsif ds == 'SV'
+      keys_candidates_list = [
+        [:USUBJID, :SVSTDTC, :SUBJID],
+        [:USUBJID, :VISITNUM, :SUBJID]
+      ]
+
+      results = keys_candidates_list.map do |keys_candidates|
+        result = look_keys(ds, keys_candidates, records)
+        if result[:valid]
+          { valid: true, key: result[:key], candidates: keys_candidates }
+        else
+          { valid: false, last_valid_key: result[:last_valid_key],
+            duplicate_file: result[:duplicate_file], candidates: keys_candidates }
+        end
+      end
+
+      # Retourner un array de résultats et valid si au moins une clé est valide
+      [results, results.any? { |r| r[:valid] }]
+
+    else
+      # Cas standard
+      keys_candidates, candidates_type = candidates(ds)
+      result = look_keys(ds, keys_candidates, records)
+
+      if result[:valid]
+        [
+          { valid: true, key: result[:key], candidates: keys_candidates},
+          true,
+          candidates_type
+        ]
+      else
+        [
+          { valid: false, last_valid_key: result[:last_valid_key],
+           duplicate_file: result[:duplicate_file], candidates: keys_candidates },
+          false,
+          candidates_type
+        ]
+      end
+    end
+  end
+
+  # create a list of the candidate variables for the key search
+  # - always starts with USUBJID
+  # - a fix list of candidate is fixed by default
+  # - some additional variables can be added via the config file
+  private def candidates(ds)
+
+    # candidate keys creation by dataset
+    # according to the dataset type
+    # event, intervention, finding
+    if %w(AE BE CE DV HO MH).include? ds
+      expected_candidates = @config['event_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'General Observation, event dataset'
+      end
+    elsif %w(CM EC EX ML PR SU).include? ds
+      expected_candidates = @config['intervention_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'General Observation, intervention dataset'
+      end
+    elsif %w(BS DA DD EG IE IS LB MB MI MK MO PC PE PF PP RP RS SC SS TR TU VS ZI).include? ds
+      expected_candidates = @config['finding_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'General Observation, finding dataset'
+      end
+    elsif %w(QS).include? ds
+      expected_candidates = %w(CAT TESTCD VISITNUM)
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'General Observation, finding dataset'
+      end
+    elsif %w(FA ZA).include? ds
+      expected_candidates = @config['finding_about_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'General Observation, finding about dataset'
+      end
+    elsif %w(DC DM).include? ds
+      expected_candidates = ['SUBJID']
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Demographic Dataset'
+      end
+    elsif %w(DS).include? ds
+      expected_candidates = @config['ds_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Special Dataset, DS'
+      end
+    elsif %w(RELREC).include? ds
+      expected_candidates = @config['relrec_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Special Dataset, RELREC'
+      end
+    elsif %w(CO).include? ds
+      expected_candidates = @config['CO_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Special Dataset, CO'
+      end
+    elsif ds == 'TA'
+      expected_candidates = @config['TA_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Trial Design Dataset, TA'
+      end
+    elsif ds == 'TE'
+      expected_candidates = @config['TE_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Trial Design Dataset, TE'
+      end
+    elsif ds == 'TI'
+      expected_candidates = @config['TI_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Trial Design Dataset, TI'
+      end
+    elsif ds == 'TS'
+      expected_candidates = @config['TS_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Trial Design Dataset, TS'
+      end
+    elsif ds == 'TV'
+      expected_candidates = @config['TV_key']&.split(' ')
+      if expected_candidates.nil?
+        rpt_cand = '- No candidates keys variables, please check the config file'
+      else
+        rpt_cand = 'Trial Design Dataset, TV'
+      end
+    elsif ds.start_with? 'SUPP'
+      expected_candidates = ['USUBJID', 'IDVAR', 'IDVARVAL', 'QNAM']
+      rpt_cand = 'SUPP dataset'
+    else
+      puts "Unknown dataset: #{ds}"
+      rpt_cand = "no key candidate for #{ds}"
+      expected_candidates = []
+    end
+
+    # no expected_candidates
+    if expected_candidates.nil?
+      expected_candidates = []
+      puts "#{ds}: No variables are specified in the configuration file for the key search."
+    end
+
+    # transform list of variables into symbol
+    # add the dataset prefix to variables when expected
+    if %w(TA TE TI TS TV RELREC CO).include? ds
+      keys_candidates = expected_candidates.map(&:to_sym)
+    elsif ds.start_with? 'SUPP'
+      keys_candidates = expected_candidates.map(&:to_sym)
+    else
+      keys_candidates = [:USUBJID,] +
+        expected_candidates.map do |c|
+          if %w(VISITNUM SUBJID).include? c
+            c.to_sym
+          else
+            "#{ds}#{c}".to_sym
+          end
+        end
+    end
+
+    # remove variable unexpected for specific dataset
+    if ds == 'TU'
+      # no visit or timepoint are expected in TU key
+      keys_candidates.delete(:VISITNUM)
+      keys_candidates.delete(:TUTPTNUM)
+    end
+
+    [keys_candidates, rpt_cand]
+  end
+
+  # look variable by variable if the key are valid
+  private def look_keys(ds, keys_candidates, records)
+    # initialize the number of duplicate with the records of records present in the dataset
+    prev_records_max = records.size
+    prev_nb_sub = records.size
+    # initialize the useless variables as an empty array
+    useless_vars = []
+    # save report to add or according to the end
+    report_key = []
+
+    # display the variables check during the search
+    report_key << "- the key search is based on the following variables in the given order: #{keys_candidates.join(', ')}"
+
+    # loop on all candidate keys variables
+    keys_candidates.size.times do |i|
+
+      # tested key
+      keys_vars = keys_candidates[..i]
+      # remove useless variables
+      useless_vars.each do |v|
+        keys_vars.delete(v)
+      end
+      report_key << "  - trying #{keys_vars.join(', ')}" if @verbose
+
+      # array of values of the variables
+      keys_records = records.group_by do |p|
+        keys_vars.map { |var| p[var] }
+      end
+
+      # get the maximum of records by key
+      records_max = 0
+      nb_sub = 0
+      keys_records.each do |values, list|
+        records_max = list.size if list.size > records_max
+        if list.size > 1
+          nb_sub += 1
+        end
+      end
+
+      # break if no duplicate
+      if records_max == 1
+        return {
+          valid: true,
+          key: keys_vars
+        }
+      end
+
+      # duplicate still present
+      report_key << "  - nope: #{nb_sub} keys variables have a maximum #{records_max} records for #{keys_vars.join(', ')}" if @verbose
+      # if the number of current duplicate is superior or equal to the previous
+      # number of duplicate then the variable is not useful for the key search and
+      # the previous number of maximum duplicate stay as is
+      # the current variable will be remove for the next search
+      if prev_records_max <= records_max && prev_nb_sub <= nb_sub
+        report_key << "  - #{keys_vars.last} not useful"
+        useless_vars << keys_vars.last
+      end
+
+      # current duplicate number become the current number of duplicate
+      prev_records_max = records_max
+      prev_nb_sub = nb_sub
+
+      # duplicate if no more variables
+      if i == (keys_candidates.size-1)
+        # duplicates records with the last valid key
+        last_valid_keys = keys_candidates.reject { |kc| useless_vars.include? kc }
+
+        # compile all duplicates with the last valid keys
+        duplicate_records = records.group_by do |p|
+          last_valid_keys.map { |var| p[var] }
+        end
+
+        # write the CSV file
+        file_name = display_duplicate_records(ds, duplicate_records, :data)
+
+        report_key << "- no more candidates variables"
+        report_key << "  - the last interesting variables checked for the key are #{last_valid_keys.join(', ')}"
+        if outname_ext == 'csv'
+          report_key << "  - all duplicates are present here: #{file_name}"
+        else
+          report_key << "  - all duplicates are present here: #{output_file}, sheet: #{File.basename(file_name, '.csv')}"
+        end
+        return {
+          valid: false,
+          last_valid_key: last_valid_keys,
+          duplicate_file: file_name
+        }
+      end
+    end
+
+  end
+
+  # search the key
+  def key_check_define(ds, keys, records)
+    keys ||= [:USUBJID]
+
+    keys_records = records.group_by do |p|
+      keys.map { |var| p[var.to_sym] }
+    end
+
+    records_max = keys_records.map { |_, list| list.size }.max || 0
+
+    if records_max == 1
+      [keys, true, nil]
+    else
+      file_name = display_duplicate_records(ds, keys_records, :define)
+      [keys, false, file_name]
+    end
+  end
+
+  # search non-ASCII character
+  def ascii_search(records)
+    # initialize list of exception
+    non_ascii_list = []
+    # read all characters
+    records.each do |record|
+      record.each do |var, value|
+        next if value.nil? || value.is_a?(Integer) || value.is_a?(Float)
+        if value.match(/[^[:ascii:]]/)
+          non_ascii_list << "#{var}: #{value.inspect}"
+        end
+      end
+    end
+
+    [non_ascii_list, non_ascii_list.empty?]
+  end
+
+  ###
+  # load config file
+  def load_config(file)
+    YAML.load_file(file)
+  end
+
+  ###
+  # load xpt file
+  def load_dsfile(infile)
+    reader = SAS::XPT::Reader.new(infile)
+
+    # only the fist dataset is loaded
+    puts "several datasets in one xpt, only the first one is kept" if reader.library.datasets.size > 1
+    dataset = reader.library.datasets.first
+
+    vars = dataset.variables.map(&:name).map(&:to_sym)
+    obs = dataset.observations
+
+    records = obs.map do |values|
+        vars.zip(values).to_h
+      end
+
+    records
+  end
+
+  ####
+  # load define.xml with Nokogiri when present
+  # export the key variables by dataset (hash)
+  # nil otherwise
+  def load_define
+    # structure for report
+    define_report = {
+      define_path: nil,
+      load: false,
+    }
+
+    # initialize from config file
+    define_path = @config['define_path']
+    # define not mentionned at the end of the path
+    if define_path != '-' && File.basename(define_path) != 'define.xml'
+      define_path = "#{define_path}/define.xml"
+    end
+    # store
+    define_report[:define_path] = define_path
+
+    # use false to distinguish
+    if define_path == '-'
+      define_report[:load] = false
+      define_node = '-'
+    else
+      begin
+        define_node = File.open(define_path, "rb") { |io| Nokogiri::XML(io, &:noblanks) }
+        define_report[:load] = true
+      rescue => e
+        puts "⚠ Error loading define.xml : #{e.message}"
+        define_report[:load] = false
+        define_node = nil
+      end
+    end
+
+    [get_define_keys(define_node), define_report]
+  end
+
+  # get the define key variables by dataset from a nokogiri node,
+  # return an hash of dataset with list of variables name
+  # - exclude supp dataset
+  # - keys order is not supported
+  private def get_define_keys(define_node)
+    # shortcut if no define
+    return nil if define_node.nil?
+    # shortcut if define = -
+    return '-' if define_node == '-'
+    # store all ItemGroup
+    item_group = define_node.css("ItemGroupDef")
+    # store all Item
+    item_def = define_node.css("ItemDef").group_by { |id| id['OID'] }
+
+    # keep only the dataset name & the key variables
+    define_keys = {}
+    # loop on all item group (dataset)
+    item_group.each do |ig|
+      key_list = []
+      ds = ig["Name"].upcase
+      # loop on all item (variables)
+      ig.css("ItemRef").each do |ir|
+        # exclude all variables not in the key
+        index = ir["KeySequence"]&.to_i
+        next if index.nil?
+        # get the ItemDef of the variable by the ItemRef OID
+        item = item_def[ir["ItemOID"]]
+        # expect one item per variable
+        unless item.size == 1
+          puts "several item with the same OID: #{item.size}"
+          item.each do |i|
+            puts i
+          end
+          puts "only the first is kept"
+        end
+        # store the variable name
+        key_list << item.first["Name"]
+      end
+
+      # assign the key to the dataset
+      define_keys[ds] = key_list
+    end
+
+    define_keys
+  end
+
+  ###
+  # display
+  #
+  # create a CVS with all duplicates
+  def display_duplicate_records(ds, keys_records, type)
+
+    if outname_ext == 'csv'
+      display_out = "#{dir_out}/#{type.to_s}_#{ds}.csv"
+    else
+      display_out = "#{dir_out}/#{type.to_s}_#{ds}.csv"
+    end
+
+    # TODO: hum ça marche ouais mais non, faut pas laisser comme ça
+    header = keys_records.first.last.first.keys
+    header = header.insert(0, 'No')
+
+    duplicate = keys_records.reject { |key, list| list.size == 1 }
+
+    CSV.open(display_out, "w") do |csv|
+      # header
+      csv << header
+      # body
+      duplicate.each do |key, list|
+        idx = duplicate.keys.index(key) + 1
+        list.each do |l|
+          row = l.values.insert(0, idx)
+          csv << row
+        end
+      end
+    end
+
+    display_out
+  end
+
+  # write report
+  def write_html_report
+    @report.output_file = output_file
+    @report.save_html
+  end
+
+  def write_word_report
+
+    # write html report unless if already exist
+    write_html_report unless File.exist?(@report.report_name)
+
+    # compute name
+    report_html = @report.report_name
+    report_word = report_html[...-5] + '.docx'
+
+    # parse html
+    puts "Parsing #{report_html}..."
+    blocks = RiReportParser.new(report_html).parse
+    puts "  -> #{blocks.size} blocks extracted"
+
+    # create word
+    puts "Building #{report_word}..."
+    DocxWriter.new(report_word).write(blocks)
+    puts "  -> Done: #{report_word}"
+  end
+
+  private def write_xls(csv_files)
+    # create xls file
+    book = Spreadsheet::Workbook.new
+
+    # workbook default format
+    format_default = Spreadsheet::Format.new(
+      font: Spreadsheet::Font.new('Verdana', size: 8)
+    )
+
+    #
+    format_gray = Spreadsheet::Format.new(
+      font: Spreadsheet::Font.new('Verdana', size: 8),
+      pattern: 1,
+      pattern_fg_color: :silver
+    )
+
+    # header format
+    format_header = Spreadsheet::Format.new(
+      weight: :bold, # TODO, not working, dont known why
+      font: Spreadsheet::Font.new('Verdana', size: 8),
+      pattern: 1,
+      pattern_fg_color: :grey
+    )
+
+    # loop on csv files
+    csv_files.each do |csv_file|
+      # create sheet name
+      sheet_name = File.basename(csv_file, '.csv')[11...]
+
+      # create worksheet
+      sheet = book.create_worksheet(name: sheet_name)
+
+      # initialize variables for the color switch
+      previous_first_cell_value = nil
+      should_color = false
+
+      # read csv & write worksheet
+      row_index = 0
+      CSV.foreach(csv_file, headers: false, encoding: 'UTF-8') do |row|
+
+        # check the first cell on the current row, to known if the color must switch
+        if row_index >= 0  # switch start after the first duplicate
+          current_first_cell_value = row[0]
+          if current_first_cell_value != previous_first_cell_value
+            should_color = !should_color  # color switch
+            previous_first_cell_value = current_first_cell_value
+          end
+        end
+
+        # set format
+        # on the header
+        if row_index == 0
+          sheet.row(row_index).default_format = format_header
+        # format alternate
+        elsif should_color
+          sheet.row(row_index).default_format = format_gray
+        else
+          sheet.row(row_index).default_format = format_default
+        end
+
+        # write line by line
+        sheet.row(row_index).concat(row)
+        row_index += 1
+      end
+
+      # adjust the column length
+      if row_index > 0
+        num_columns = sheet.row(0).size
+
+        (0...num_columns).each do |col_index|
+          max_length = 10  # min length
+
+          # loop on all column
+          (0...row_index).each do |r|
+            cell_value = sheet.row(r)[col_index].to_s
+            cell_length = cell_value.length + 2  # +2 for margin
+            max_length = cell_length if cell_length > max_length
+          end
+
+          # max length is 50
+          sheet.column(col_index).width = [max_length, 50].min
+        end
+      end
+    end
+
+    # save the workbook
+    book.write(output_file)
+  end
+
+  private def write_xlsx(csv_files)
+    # create workbook
+    workbook = RubyXL::Workbook.new
+
+    # delete default worksheet
+    workbook.worksheets.delete_at(0)
+
+    # loop on all csv files
+    csv_files.each do |csv_file|
+      # create sheet name
+      sheet_name = File.basename(csv_file, '.csv')
+
+      # create worksheet
+      worksheet = workbook.add_worksheet(sheet_name)
+
+      # initialize variables for background color switch
+      previous_first_cell_value = nil
+      should_color = false
+
+      # read csv & write worksheet
+      row_index = 0
+      CSV.foreach(csv_file, headers: false, encoding: 'UTF-8') do |row|
+        row.each_with_index do |cell_value, col_index|
+
+          # check the first cell on the current row, to known if the color must switch
+          if row_index >= 0  # start with a blank line after the header
+            current_first_cell_value = row[0]
+            if current_first_cell_value != previous_first_cell_value
+              should_color = !should_color  # color switch
+              previous_first_cell_value = current_first_cell_value
+            end
+          end
+
+          # get the cell
+          cell = worksheet.add_cell(row_index, col_index, cell_value)
+
+          # set format
+          # on header row
+          if row_index == 0
+            cell.change_font_bold(true)
+            cell.change_font_name('Verdana')
+            cell.change_font_size(8)
+            cell.change_fill('C0C0C0')
+
+          # on other rows
+          else
+            cell.change_font_name('Verdana')
+            cell.change_font_size(8)
+            cell.change_fill('DCDCDC') if should_color
+          end
+        end
+
+        row_index += 1
+      end
+
+      # adjust the column length
+      if row_index > 0
+        num_columns = worksheet[0].cells.length
+
+        (0...num_columns).each do |col_index|
+          max_length = 10  # minimum length
+
+          # loop on all rows in a column
+          (0...row_index).each do |r|
+            cell = worksheet[r][col_index]
+            if cell && cell.value
+              cell_length = cell.value.to_s.length + 2  # +2 for margin
+              max_length = cell_length if cell_length > max_length
+            end
+          end
+
+          # column length is 50 or the length computed
+          col = worksheet.cols.get_range(col_index)
+          col.width = [max_length * 1.2, 50].min
+          col.custom_width = true
+        end
+      end
+    end
+
+    # save the workbook
+    workbook.write(output_file)
+  end
+
+  ###
+  # Helpers
+
+  # output directory
+  def prepare_output_dir
+    dir = "#{study_name}/#{config['output_directory']}"
+
+    # no value, current repository
+    if dir.nil? || dir.strip.empty?
+      dir = '.'
+
+    # repository present, delete all csv file
+    elsif Dir.exist?(dir)
+      Dir.glob("#{dir}/**/*.csv").each { |f| File.delete(f) }
+
+    # no repository, creation
+    else
+      FileUtils.mkdir_p(dir)
+    end
+
+    dir
+  end
+
+  def dir_out
+    dir = "#{study_name}/#{config['output_directory']}"
+    dir = '.' if dir.nil? || dir.strip.empty?
+    dir
+  end
+
+  # extension output file
+  def outname_ext
+    if %w(xls xlsx).include? config['output_type']
+      config['output_type']
+    else
+      'csv'
+    end
+  end
+
+  # compute the file name for Excel file
+  def output_file
+    if outname_ext == 'csv'
+      nil
+    else
+      "duplicates.#{outname_ext}"
+    end
+  end
+
+  # list of invalid dataset, whatever the issue
+  def dslist_invalid
+    @ds_with_issue.values.flatten.uniq
+  end
+
+  ###
+  # Report class
+  class Report
+    attr_reader :report_name
+    attr_reader :study_name
+    attr_reader :externe_file_type
+
+    attr_accessor :init_reports
+    attr_accessor :dataset_reports
+    attr_accessor :output_file
+
+    def initialize(report_name, study_name, externe_file_type)
+      @report_name = report_name
+      @study_name = study_name
+      @externe_file_type = externe_file_type
+      @init_reports = {}
+      @dataset_reports = {}
+    end
+
+    # summary part from FindKeys.initialize
+    def add_init(text_hash)
+      @init_reports = text_hash
+    end
+
+    # report part from FindKeys.high_level_check
+    def add_dataset_report(ds_name, report_data)
+      @dataset_reports[ds_name] = report_data
+    end
+
+    # dynamic title for acces web or manual
+    def title
+      if @study_name.include?('/')
+        File.basename(@study_name)
+      else
+        @study_name
+      end
+    end
+
+    # html build from ERB file
+    def generate_html
+      # Charge ERB template
+      template_path = File.expand_path('../../../views/report_template.erb', __FILE__)
+
+      unless File.exist?(template_path)
+        raise "Template file not found: #{template_path}"
+      end
+
+      # Encode logo in base64
+      logo_path = File.expand_path('../../../public/Contact-LOGO.png', __FILE__)
+      @logo_base64 = if File.exist?(logo_path)
+          require 'base64'
+          "data:image/png;base64,#{Base64.strict_encode64(File.binread(logo_path))}"
+        else
+          nil
+        end
+
+      # Create HTML
+      template = File.read(template_path)
+      erb = ERB.new(template, trim_mode: '-')
+      erb.result(binding)
+    end
+
+    # write html file
+    def save_html(output_path = nil)
+      output_path ||= @report_name
+      html_content = generate_html
+
+      File.write(output_path, html_content)
+      puts "Rapport HTML généré : #{output_path}"
+    end
+  end
+
+end
+end
+
+# config_file = 'config.yaml'
+# fk = FindKeys.new(config_file, verbose: true)
+# fk.high_level_check
+# fk.write_html_report
+# fk.write_word_report
+# puts fk.ds_with_issue
+# puts fk.dslist_invalid