hlsv 1.0.0 → 2.0.0

data/lib/hlsv/path_guard.rb

@@ -0,0 +1,32 @@
+#######
+# Copyright (c) 2026 AdClin
+# Licensed under the GNU General Public License v3.0
+#######
+
+# frozen_string_literal: true
+
+module Hlsv
+  # Path-traversal guards, meant to be included into Sinatra's `helpers do ... end`
+  # block so `halt` is available.
+  module PathGuard
+    # Basic anti-traversal check for a user-supplied relative path.
+    # Use this when the target base directory varies (e.g. a user-configured
+    # data_directory) and can't be constrained to a single root.
+    def safe_relative_path!(relative, label: "file")
+      halt 400, "Missing #{label} parameter" if relative.nil? || relative.to_s.empty?
+      halt 403, "Access denied" if relative.include?('..') || relative.start_with?('/')
+      relative
+    end
+
+    # Stronger check: resolves the path and ensures it stays within base_dir.
+    # Use this for anything rooted under a known folder (e.g. hlsv_results/).
+    def resolve_within!(base_dir, relative)
+      safe_relative_path!(relative)
+
+      base = File.expand_path(base_dir)
+      full = File.expand_path(File.join(base, relative))
+      halt 403, "Access denied" unless full.start_with?("#{base}#{File::SEPARATOR}")
+      full
+    end
+  end
+end

data/lib/hlsv/sdtm_validation/dataset.rb

@@ -0,0 +1,169 @@
+module Hlsv
+class SdtmValidation
+class Dataset
+
+  def self.from_xpt(xpt_path)
+
+    reader = SAS::XPT::Reader.new(xpt_path)
+
+    # only the fist dataset is loaded
+    puts "several datasets in one xpt, only the first one is kept" if reader.library.datasets.size > 1
+    dataset = reader.library.datasets.first
+
+    # create records as hash :variable => value
+    vars = dataset.variables.map(&:name).map(&:to_sym)
+    obs = dataset.observations
+
+    records = obs.map do |values|
+        vars.zip(values).to_h
+      end
+
+    Dataset.new(dataset.name, dataset.label, records)
+  end
+
+  attr_reader :name
+  attr_reader :label
+  attr_reader :records
+
+  attr_accessor :non_ascii_values
+
+  attr_accessor :define_status
+  attr_accessor :define_keys
+  attr_accessor :define_duplicate_records
+  attr_accessor :define_duplicate_file
+
+  attr_accessor :tested_key
+  attr_accessor :minimal_key_status
+  attr_accessor :minimal_key
+  attr_accessor :data_duplicate_records
+  attr_accessor :data_duplicate_file
+
+  def initialize(name, label, records)
+
+    @name = name
+    @label = label
+    @records = records
+
+    @non_ascii_values = nil
+    @define_status = nil
+    @define_keys = nil
+    @define_duplicate_records = nil
+    @define_duplicate_file = nil
+
+    @tested_key = nil
+    @minimal_key = []
+    @minimal_key_status = []
+    @data_duplicate_records = []
+    @data_duplicate_file = []
+
+  end
+
+  def candidates_keys(ds, config)
+    @candidates_keys ||= begin
+      keys = candidates(ds, config)
+      keys.first.is_a?(Array) ? keys : [keys]
+    end
+  end
+
+  private def candidates(ds, config)
+
+    keys = [:USUBJID]
+
+    # get the variable from the config file according to the type of dataset
+    case type(ds)
+    when :intervention, :event, :finding, :finding_about
+      keys.concat(config["#{@type}_key"]&.split&.map { |v| (%w[STUDYID DOMAIN SPDEVID POOLID VISITNUM VISIT TAETORD SUBJID].include?(v) ? v : "#{ds}#{v}").to_sym } || [])
+    when :trial_design
+      keys.concat(config["#{ds}_key"]&.split || [])
+    when :relation
+      keys.concat(config['RELREC_key']&.split || [])
+    when :supp
+      keys = %w(USUBJID IDVAR IDVARVAL QNAM).map(&:to_sym)
+    when :special
+      if ds == 'CO'
+        keys.concat(config['CO_key']&.split || [])
+      elsif ds == 'DS'
+        keys.concat(config['DS_key']&.split || [])
+      elsif %w(DM DC).include? ds
+        keys = [:USUBJID, :SUBJID]
+      elsif ds == 'SE'
+        keys = [[:USUBJID, :EPOCH, :SUBJID], [:USUBJID, :TAETORD, :SUBJID]]
+      elsif ds == 'SV'
+        keys = [[:USUBJID, :SVSTDTC, :SUBJID], [:USUBJID, :VISITNUM, :SUBJID]]
+      end
+    else
+      puts "/!\\ No candidates keys: #{@type}"
+    end
+
+    puts "/!\\ No candidates keys: #{@type}" if keys == [:USUBJID]
+    @tested_key = keys.first.is_a?(Array) ? keys : [keys]
+
+    return keys
+  end
+
+  INTERVENTION = %w(AG CM EC EX ML PR SU)
+  EVENT = %w(AE BE CE DV HO MH)
+  FINDING = %w(BS CP GF IS LB MB MI MS PC PP CV MK NV OE RE RP UR GI IG FT QS RS DA DD EG IE PE SC
+              SS TR TU VS)
+  FINDING_ABOUT = %w(FA SR)
+  TRIAL_DESIGN = %w(TA TD TE TI TM TS TV TX)
+  RELATION = %w(RELREC RELSPEC RELSUB)
+  SPECIAL = %w(CO DM SE SV DS)
+
+  def type(ds)
+
+    @type ||= case ds
+      when *INTERVENTION
+        :intervention
+      when *EVENT
+        :event
+      when *FINDING
+        :finding
+      when *FINDING_ABOUT
+        :finding_about
+      when *TRIAL_DESIGN
+        :trial_design
+      when *RELATION
+        :relation
+      when *SPECIAL
+        :special
+      when /\ASUPP/
+        :supp
+      else
+        puts "/!\\ Unexpected dataset: #{ds}"
+      end
+  end
+
+  def type_to_s(ds)
+    @type_to_s ||= case type(ds)
+      when :intervention, :event, :finding, :finding_about
+        "General Oberservation, #{@type} dataset"
+      when :relation
+        "Relationship Dataset, #{ds}"
+      when :trial_design
+        "Trial Design, #{ds}"
+      when :special
+        "Special Dataset, #{ds}"
+      when :supp
+        "Supp Dataset"
+      else
+        puts "/!\\ Unexpected dataset: #{ds}, type: #{@type}"
+
+        "Undefined"
+      end
+  end
+
+  # when some non ascii charactere have been found, then status is false
+  # otherwiese status is true
+  def ascii_status
+    if non_ascii_values.nil?
+      nil
+    elsif non_ascii_values.empty?
+      true
+    else
+      false
+    end
+  end
+end
+end
+end

data/lib/hlsv/sdtm_validation/define.rb

@@ -0,0 +1,142 @@
+module Hlsv
+class SdtmValidation
+class Define
+
+  attr_reader :path        # String
+  attr_reader :status      # true/false/nil
+  attr_reader :keys        # Hash dataset: variables (Array of String)
+  attr_reader :exception   # Array of String
+
+  def initialize(infile)
+    @exception = []
+
+    @path = check_path(infile)
+
+    node, @status = load_define
+
+    @keys = get_define_keys(node)
+
+  end
+
+  def check_path(path)
+
+    if path.nil? || path == '-'
+      path
+    elsif File.basename(path) == 'define.xml'
+      path
+    else
+
+      if File.directory? path
+        @exception << "The path isn't complete, it's a folder."
+        @exception << "To find the file, add 'define.xml' to the folder in the config (already done here)."
+        puts 'The define path is not complete.'
+
+        "#{path}/define.xml"
+      elsif File.exist? path
+        path
+      else
+        @exception << "The file cannot be read: #{path.inspect}"
+
+        nil
+      end
+    end
+  end
+
+  # load define from the config information
+  # return Nokogiri::Node
+  def load_define
+
+    # no define specify: define analyse part, not run with error
+    if @path.nil?
+      puts "no define specify"
+      node = nil
+      status = nil
+
+    # no define expected: define analyse part, not run
+    elsif @path == '-'
+      puts "no define expected"
+      node = '-'
+      status = false
+
+    # define present
+    else
+      begin
+        node = File.open(@path, "rb") { |io| Nokogiri::XML(io, &:noblanks) }
+        status = true
+        puts "define.xml loaded"
+
+      rescue => e
+        puts "⚠ Error loading define.xml : #{e.message}"
+        node = nil
+        status = nil
+        @exception << 'Invalid path, update the config'
+      end
+    end
+
+    [node, status]
+  end
+
+  def datasets
+    if keys.nil? || keys == '-'
+      []
+    else
+      keys.keys
+    end
+  end
+
+  def define_key_for(ds)
+    if keys.nil? || keys == '-'
+      []
+    else
+      keys[ds]
+    end
+  end
+
+  # get the define key variables by dataset from a nokogiri node,
+  # return an hash of dataset with list of variables name
+  # - keys order is not supported
+  private def get_define_keys(define_node)
+    # shortcut if no define
+    return nil if define_node.nil?
+    # shortcut if define = -
+    return '-' if define_node == '-'
+    # store all ItemGroup
+    item_group = define_node.css("ItemGroupDef")
+    # store all Item
+    item_def = define_node.css("ItemDef").group_by { |id| id['OID'] }
+
+    # keep only the dataset name & the key variables
+    define_keys = {}
+    # loop on all item group (dataset)
+    item_group.each do |ig|
+      key_list = []
+      ds = ig["Name"].upcase
+      # loop on all item (variables)
+      ig.css("ItemRef").each do |ir|
+        # exclude all variables not in the key
+        index = ir["KeySequence"]&.to_i
+        next if index.nil?
+        # get the ItemDef of the variable by the ItemRef OID
+        item = item_def[ir["ItemOID"]]
+        # expect one item per variable
+        unless item.size == 1
+          puts "several item with the same OID: #{item.size}"
+          item.each do |i|
+            puts i
+          end
+          puts "only the first is kept"
+        end
+        # store the variable name
+        key_list << item.first["Name"]
+      end
+
+      # assign the key to the dataset
+      define_keys[ds] = key_list
+    end
+
+    define_keys
+  end
+
+end
+end
+end

data/lib/hlsv/sdtm_validation/report.rb

@@ -0,0 +1,357 @@
+require 'uri'
+require 'erb'
+require 'fast_excel'
+
+require_relative '../url_helper'
+
+module Hlsv
+class SdtmValidation
+class Report
+
+  include Hlsv::UrlHelper
+
+  attr_reader :study_name
+  attr_reader :ds_path
+  attr_reader :datasets
+  attr_reader :define
+  attr_reader :excluded_dataset
+
+  attr_reader :report_name
+
+  REPORT_SUFFIX = '_high_level_check'
+
+  # Single source of truth for the report base path, so callers (web_app.rb)
+  # can check whether a report already exists on disk without instantiating
+  # a full Report (which requires datasets/define in memory)
+  def self.report_name_for(study_name)
+    "#{study_name}/#{File.basename(study_name).gsub(' ', '_').downcase}#{REPORT_SUFFIX}"
+  end
+
+  # Builds a standalone one-sheet Excel workbook for a single duplicate CSV
+  # file already on disk - used by the "Export to Excel" button on
+  # /csv_view. Unlike excel_export/excel_export_folder, this never needs the
+  # analysis in memory: everything required (the CSV path, the dataset name,
+  # the tested key) is already carried by the static HTML report's query
+  # string, so this works even after a server restart.
+  def self.build_single_csv_workbook(study_name:, ds_name:, csv_path:, key: nil)
+    report = allocate
+    report.instance_variable_set(:@study_name, study_name)
+    report.send(:build_single_csv_workbook_body, ds_name: ds_name, csv_path: csv_path, key: key)
+  end
+
+  def initialize(study_name, ds_path, datasets, define, excluded_dataset, web_mode: false)
+
+    @web_mode = web_mode
+    @study_name = study_name
+    @ds_path = ds_path
+    @datasets = datasets
+    @define = define
+    @excluded_dataset = excluded_dataset
+
+    @report_name = self.class.report_name_for(@study_name)
+
+    puts "\n=== Report"
+  end
+
+  # dynamic title for acces web or manual
+  def title
+    if @study_name.include?('/')
+      File.basename(@study_name)
+    else
+      @study_name
+    end
+  end
+
+  def config_file_path
+    "#{study_name}/config.yaml"
+  end
+
+  def relative_report_path(file)
+    file.sub("#{@study_name}/", '')
+  end
+
+  ###
+  # HTML section
+  ###
+
+  # html build from ERB file
+  def generate_html
+    # Charge ERB template
+    # template_path = File.expand_path('../../../../views/report_template.erb', __FILE__)
+    template_path = File.expand_path('../../../../views/report.erb', __FILE__)
+
+    unless File.exist?(template_path)
+      raise "Template file not found: #{template_path}"
+    end
+
+    # Encode logo in base64
+    logo_path = File.expand_path('../../../../public/Contact-LOGO.png', __FILE__)
+    @logo_base64 = if File.exist?(logo_path)
+        require 'base64'
+        "data:image/png;base64,#{Base64.strict_encode64(File.binread(logo_path))}"
+      else
+        nil
+      end
+
+    # Create HTML
+    template = File.read(template_path)
+    erb = ERB.new(template, trim_mode: '-')
+    erb.result(binding)
+  end
+
+  ###
+  # Word section
+  ###
+
+  # write html file
+  def html
+    output_path = "#{@report_name}.html"
+    html_content = generate_html
+
+    File.write(output_path, html_content)
+    puts "HTML report generated: #{output_path}"
+  end
+
+  # write docx file
+  def docx
+    html_path = "#{@report_name}.html"
+    docx_path = "#{@report_name}.docx"
+
+    # generate html if not already done
+    html unless File.exist?(html_path)
+
+    blocks = RiReportParser.new(html_path).parse
+    DocxWriter.new(docx_path).write(blocks)
+    puts "Word report generated: #{docx_path}"
+  end
+
+  ###
+  # Excel section
+  ###
+  def excel(dataset: nil)
+    output_path = dataset \
+      ? "#{File.dirname(@report_name)}/#{dataset}.xlsx"
+      : "#{@report_name}.xlsx"
+
+    kill_excel_holding(output_path)
+    wait_for_file_release(output_path)
+
+    workbook = dataset ? excel_export(dataset) : excel_export_folder
+    return unless workbook  # cas "no duplicates" dans excel_export
+
+    File.binwrite(output_path, workbook.read_string)
+    puts "Excel report generated: #{output_path}"
+  end
+
+  def excel_export(dataset)
+    ds = datasets.fetch(dataset) { raise Hlsv::Error, "Excel report: dataset not found, #{dataset.inspect}" }
+
+    if ds.data_duplicate_file.empty? && ds.define_duplicate_file.to_s.empty?
+      puts "Excel report: no duplicates present in #{dataset}, no excel file generated"
+      return nil
+    end
+
+    build_workbook([ds])
+  end
+
+  def excel_export_folder
+    ds_with_issues = @datasets.values.select { |ds|
+      ds.data_duplicate_file.any? || !ds.define_duplicate_file.to_s.empty?
+    }
+    build_workbook(ds_with_issues, extra_info: ["• Number of Datasets: #{ds_with_issues.size}"])
+  end
+
+  private
+
+  def build_single_csv_workbook_body(ds_name:, csv_path:, key:)
+    workbook = FastExcel.open(constant_memory: true)
+    formats  = build_formats(workbook)
+
+    data_keys = key ? [[ds_name, key.split(',').map(&:strip)]] : []
+
+    add_readme_sheet(workbook, formats,
+      title: "Duplicates Analysis Report - #{@study_name}",
+      info: ["• Dataset: #{ds_name}", "• Generated: #{Time.now.strftime('%Y-%m-%d at %H:%M')}"],
+      keys: [data_keys, []],
+      next_steps: [
+        "→ Review duplicate records and identify variables to add in the configuration.",
+        "→ Highlight issues with data cleaning."
+      ]
+    )
+
+    csv = CSV.read(csv_path, headers: true)
+    add_data_sheet(workbook, formats, File.basename(csv_path, '.csv')[0..30], csv) unless csv.empty?
+
+    workbook
+  end
+
+  def extract_data_keys(ds_list)
+    ds_list.flat_map do |ds|
+      if ds.minimal_key.size == 1
+        [[ds.name, ds.minimal_key.first]]
+      else
+        ds.minimal_key.each_with_index.map do |key, i|
+          ["#{ds.name} (option #{i + 1})", key]
+        end
+      end
+    end
+  end
+
+  def extract_define_keys(ds_list)
+    ds_list.filter_map { |ds| [ds.name, ds.define_keys] if !ds.define_status && !ds.define_keys.nil? }
+  end
+
+  def collect_csv_files(ds_list)
+    ds_list.flat_map { |ds| ds.data_duplicate_file + [ds.define_duplicate_file] }.compact
+  end
+
+  def build_workbook(ds_list, extra_info: [])
+    data_keys   = extract_data_keys(ds_list)
+    define_keys = extract_define_keys(ds_list)
+    csv_files   = collect_csv_files(ds_list)
+
+    workbook = FastExcel.open(constant_memory: true)
+    formats  = build_formats(workbook)
+
+    add_readme_sheet(workbook, formats,
+      title: "Duplicates Analysis Report - #{@study_name}",
+      info: ["• Folder: #{@study_name}", *extra_info, "• Generated: #{Time.now.strftime('%Y-%m-%d at %H:%M')}"],
+      keys: [data_keys, define_keys],
+      next_steps: [
+        "→ Each tab corresponds to one CSV file from the folder.",
+        "→ Review duplicate records and identify variables to add in the configuration.",
+        "→ Highlight issues with data cleaning."
+      ]
+    )
+
+    csv_files.each do |csv_file|
+      sheet_name = File.basename(csv_file, '.csv')[0..30]
+      csv = CSV.read(csv_file, headers: true)
+      next if csv.empty?
+      add_data_sheet(workbook, formats, sheet_name, csv)
+    end
+
+    workbook
+  end
+
+  def kill_excel_holding(filepath)
+    return unless Gem.win_platform?
+
+    filename = File.basename(filepath, '.*').downcase
+
+    ps_command = "Get-Process excel -ErrorAction SilentlyContinue | " \
+                 "Where-Object { $_.MainWindowTitle -like '*#{filename}*' }"
+
+    result = `powershell -NoProfile -Command "#{ps_command}"`.strip
+
+    if result.empty?
+      # Excel not running or file not open in Excel
+      return
+    end
+
+    `powershell -NoProfile -Command "Stop-Process -Name excel -Force -ErrorAction SilentlyContinue"`
+    puts "Excel closed (#{filepath} was open)"
+  end
+
+  def wait_for_file_release(filepath, max_attempts: 10, wait: 1)
+    return unless File.exist?(filepath)
+
+    max_attempts.times do |i|
+      begin
+        File.open(filepath, "r+b") { }
+        return # File is released, we can proceed
+      rescue Errno::EACCES, Errno::EBUSY
+        puts "Waiting for file release... (#{i + 1}/#{max_attempts})"
+        sleep wait
+      end
+    end
+
+    raise "Could not access file after #{max_attempts} attempts: #{filepath}"
+  end
+
+  def build_formats(workbook)
+    {
+      title:   workbook.add_format(bold: true, font_size: 16, font_color: "#2c3e50", align: "left"),
+      heading: workbook.add_format(bold: true, font_size: 12, font_color: "#2c3e50", bg_color: "#e8f4f8", pattern: 1, border: 1),
+      text:    workbook.add_format(font_size: 11, align: "left", text_wrap: true),
+      footer:  workbook.add_format(font_size: 9, font_color: "#6c757d", align: "center"),
+      header:  workbook.add_format(bold: true, bg_color: "#c0c0c0", pattern: 1, border: 1),
+      even:    workbook.add_format(bg_color: "#FFFFFF", pattern: 1, border: 1),
+      odd:     workbook.add_format(bg_color: "#E3F2FD", pattern: 1, border: 1)
+    }
+  end
+
+  def add_readme_sheet(workbook, formats, title:, info:, keys:, next_steps:)
+    sheet = workbook.add_worksheet("README")
+    sheet.set_column(0, 0, 80)
+    sheet.set_column(1, 1, 20)
+
+    row = 0
+    sheet.write_string(row, 0, title, formats[:title])
+    row += 2
+
+    sheet.write_string(row, 0, "About This Report", formats[:heading])
+    row += 1
+    [
+      "This Excel file displays the detected duplicates, grouped according to the last key tested.",
+      "The duplicate groups are represented by a number in the 'No' column.",
+      "Alternating colors (white and light blue) are used to distinguish them visually.",
+      "All variables present in the dataset are displayed."
+    ].each { |line| sheet.write_string(row, 0, line, formats[:text]); row += 1 }
+    row += 1
+
+    sheet.write_string(row, 0, "Sheet Information", formats[:heading])
+    row += 1
+    info.each { |line| sheet.write_string(row, 0, line, formats[:text]); row += 1 }
+    row += 1
+    sheet.write_string(row, 0, "Keys Information", formats[:heading])
+    row += 1
+    unless keys[0].empty?
+    sheet.write_string(row, 0, "data", formats[:text]); row += 1
+      keys[0].each do |ds, vars|
+        next if vars.nil?
+        sheet.write_string(row, 0, "• #{ds}: #{vars.join(', ')}", formats[:text])
+        row += 1
+      end
+    end
+    unless keys[1].empty?
+      sheet.write_string(row, 0, "define", formats[:text]); row += 1
+      keys[1].each do |ds, vars|
+        next if vars.nil?
+        sheet.write_string(row, 0, "• #{ds}: #{vars.join(', ')}", formats[:text])
+        row += 1
+      end
+    end
+    row += 1
+    sheet.write_string(row, 0, "Next Steps", formats[:heading])
+    row += 1
+    next_steps.each { |line| sheet.write_string(row, 0, line, formats[:text]); row += 1 }
+    row += 1
+
+    sheet.write_string(row, 0,
+      "© #{Time.now.year} AdClin. All rights reserved. | Licensed under AGPL v3",
+      formats[:footer])
+  end
+
+  def add_data_sheet(workbook, formats, sheet_name, csv)
+    sheet = workbook.add_worksheet(sheet_name)
+    col_widths = csv.headers.map { |h| h.to_s.length }
+
+    csv.headers.each_with_index { |h, col| sheet.write_string(0, col, h.to_s, formats[:header]) }
+    sheet.freeze_panes(1, 5)
+
+    csv.each_with_index do |row, row_index|
+      fmt = row[0].to_i.even? ? formats[:even] : formats[:odd]
+      row.fields.each_with_index do |value, col|
+        value_str = value.to_s
+        sheet.write_string(row_index + 1, col, value_str, fmt)
+        col_widths[col] = [col_widths[col], value_str.length].max
+      end
+    end
+
+    col_widths.each_with_index { |width, col| sheet.set_column(col, col, width + 2) }
+  end
+
+end
+end
+end