hlsv 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +44 -3
- data/README.md +36 -29
- data/lib/hlsv/analysis_registry.rb +47 -0
- data/lib/hlsv/analysis_runner.rb +193 -0
- data/lib/hlsv/config_manager.rb +125 -0
- data/lib/hlsv/html2word.rb +18 -18
- data/lib/hlsv/path_guard.rb +32 -0
- data/lib/hlsv/sdtm_validation/dataset.rb +169 -0
- data/lib/hlsv/sdtm_validation/define.rb +142 -0
- data/lib/hlsv/sdtm_validation/report.rb +357 -0
- data/lib/hlsv/sdtm_validation.rb +390 -0
- data/lib/hlsv/url_helper.rb +28 -0
- data/lib/hlsv/version.rb +1 -1
- data/lib/hlsv/web_app.rb +264 -418
- data/lib/hlsv.rb +12 -5
- data/public/css/accessibility/accessibility.css +33 -0
- data/public/css/base/layout.css +33 -0
- data/public/css/base/reset.css +23 -0
- data/public/css/base/typography.css +31 -0
- data/public/css/components/buttons.css +142 -0
- data/public/css/components/file-tree.css +107 -0
- data/public/css/components/footer.css +43 -0
- data/public/css/components/forms.css +56 -0
- data/public/css/components/header.css +52 -0
- data/public/css/components/status.css +56 -0
- data/public/css/features/csv-table.css +204 -0
- data/public/css/features/file-browser.css +208 -0
- data/public/css/responsive/responsive.css +133 -0
- data/public/css/styles.css +25 -0
- data/public/css/styles_csv.css +23 -0
- data/public/favicon.ico +0 -0
- data/public/js/analysis.js +201 -0
- data/public/js/app.js +63 -0
- data/public/js/browser.js +172 -0
- data/public/js/config.js +214 -0
- data/public/js/results.js +240 -0
- data/public/js/utils.js +57 -0
- data/views/csv_view.erb +11 -12
- data/views/index.erb +70 -19
- data/views/{report_template.erb → report.erb} +203 -188
- metadata +39 -41
- data/lib/hlsv/find_keys.rb +0 -979
- data/lib/hlsv/mon_script.rb +0 -169
- data/public/app.js +0 -569
- data/public/styles.css +0 -586
- data/public/styles_csv.css +0 -448
data/lib/hlsv/web_app.rb
CHANGED
|
@@ -12,173 +12,213 @@ require 'fast_excel'
|
|
|
12
12
|
require 'zip'
|
|
13
13
|
require 'nokogiri'
|
|
14
14
|
|
|
15
|
+
require_relative 'config_manager'
|
|
16
|
+
require_relative 'path_guard'
|
|
17
|
+
require_relative 'url_helper'
|
|
18
|
+
require_relative 'analysis_registry'
|
|
19
|
+
|
|
15
20
|
module Hlsv
|
|
16
21
|
class WebApp < Sinatra::Base
|
|
17
22
|
|
|
18
23
|
set :root, Hlsv::INSTALL_ROOT
|
|
19
24
|
|
|
25
|
+
RESULTS_DIR = 'hlsv_results'
|
|
26
|
+
|
|
20
27
|
# ---------------------------------------------------------------------------
|
|
21
|
-
# :section: ROUTES
|
|
28
|
+
# :section: ROUTES - Views
|
|
22
29
|
# ---------------------------------------------------------------------------
|
|
23
30
|
|
|
24
31
|
# Homepage: loads config
|
|
25
32
|
get '/' do
|
|
26
|
-
@config =
|
|
33
|
+
@config = ConfigManager.load
|
|
27
34
|
erb :index
|
|
28
35
|
end
|
|
29
36
|
|
|
30
37
|
# CSV viewer: displays a CSV file as an HTML table
|
|
31
38
|
# Params:
|
|
32
|
-
# :file
|
|
33
|
-
# :last_valid_key
|
|
39
|
+
# :file - path to the CSV file
|
|
40
|
+
# :last_valid_key - comma-separated list of last tested keys (optional)
|
|
41
|
+
# :study - study name, used to build the "Export to Excel" link (optional)
|
|
42
|
+
# :dataset - dataset name, used to build the "Export to Excel" link (optional,
|
|
43
|
+
# falls back to parsing the filename if not provided)
|
|
34
44
|
get '/csv_view' do
|
|
35
|
-
file = params[:file]
|
|
36
|
-
halt
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
type_dup
|
|
42
|
-
@
|
|
43
|
-
@
|
|
44
|
-
@
|
|
45
|
+
file = safe_relative_path!(params[:file])
|
|
46
|
+
halt 404, "File not found" unless File.exist?(file)
|
|
47
|
+
|
|
48
|
+
@file = file
|
|
49
|
+
csv_name = File.basename(file, '.csv')
|
|
50
|
+
type_dup = csv_name.split('_').first
|
|
51
|
+
@type = type_dup == 'data' ? 'dataset' : 'define.xml'
|
|
52
|
+
@ds_name = params[:dataset] || csv_name.split('_').last
|
|
53
|
+
@study = params[:study]
|
|
54
|
+
@last_valid_key = params[:last_valid_key]&.split(',') || []
|
|
55
|
+
@rows = CSV.read(file, headers: true)
|
|
45
56
|
|
|
46
57
|
erb :csv_view
|
|
47
58
|
end
|
|
48
59
|
|
|
49
60
|
# ---------------------------------------------------------------------------
|
|
50
|
-
# :section: ROUTES
|
|
61
|
+
# :section: ROUTES - Configuration (JSON API)
|
|
51
62
|
# ---------------------------------------------------------------------------
|
|
52
63
|
|
|
53
|
-
# GET /config
|
|
64
|
+
# GET /config - Returns the current configuration as JSON
|
|
54
65
|
get '/config' do
|
|
55
66
|
content_type :json
|
|
56
|
-
|
|
67
|
+
ConfigManager.load.to_json
|
|
57
68
|
end
|
|
58
69
|
|
|
59
|
-
# POST /config
|
|
70
|
+
# POST /config - Updates editable fields in config.yaml
|
|
60
71
|
# Body: JSON object with field/value pairs
|
|
61
72
|
post '/config' do
|
|
62
|
-
|
|
63
|
-
begin
|
|
73
|
+
json_action do
|
|
64
74
|
config_params = JSON.parse(request.body.read)
|
|
65
|
-
|
|
66
|
-
{
|
|
67
|
-
rescue => e
|
|
68
|
-
status 500
|
|
69
|
-
{ success: false, erreur: e.message }.to_json
|
|
75
|
+
ConfigManager.save(config_params)
|
|
76
|
+
{ message: "Configuration updated successfully" }
|
|
70
77
|
end
|
|
71
78
|
end
|
|
72
79
|
|
|
73
|
-
# POST /config/reset
|
|
80
|
+
# POST /config/reset - Reloads config.yaml from config.default.yaml
|
|
74
81
|
post '/config/reset' do
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
82
|
+
json_action do
|
|
83
|
+
ConfigManager.reset
|
|
84
|
+
{ message: "Default configuration loaded from config.default.yaml" }
|
|
85
|
+
end
|
|
86
|
+
end
|
|
80
87
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
88
|
+
# POST /config/load_existing - Loads config.yaml from an arbitrary file
|
|
89
|
+
# picked through the config page's file browser (GET /browse). Same
|
|
90
|
+
# trust boundary as /browse: the user explicitly chose this exact file
|
|
91
|
+
# via the picker, so no extra path restriction is applied here.
|
|
92
|
+
# Body: { "path": "/absolute/path/to/some_config.yaml" }
|
|
93
|
+
post '/config/load_existing' do
|
|
94
|
+
json_action do
|
|
95
|
+
payload = JSON.parse(request.body.read)
|
|
96
|
+
path = payload['path'].to_s.strip
|
|
97
|
+
|
|
98
|
+
raise "Missing path parameter" if path.empty?
|
|
99
|
+
|
|
100
|
+
ConfigManager.load_from(path)
|
|
101
|
+
{ message: "Configuration loaded from #{File.basename(path)}" }
|
|
87
102
|
end
|
|
88
103
|
end
|
|
89
104
|
|
|
90
|
-
# POST /config/clear
|
|
105
|
+
# POST /config/clear - Resets all config fields to nil (keeps structure)
|
|
91
106
|
post '/config/clear' do
|
|
107
|
+
json_action do
|
|
108
|
+
ConfigManager.clear
|
|
109
|
+
{ message: "Configuration cleared" }
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# GET /browse - Lists directory entries for the config page's "Browse"
|
|
114
|
+
# buttons (Datasets Directory / Path to define.xml). This intentionally
|
|
115
|
+
# lets the user navigate anywhere on disk - the same trust boundary as
|
|
116
|
+
# typing an absolute path into the field by hand, since this is a
|
|
117
|
+
# local-only app. safe_relative_path!/resolve_within! don't apply here:
|
|
118
|
+
# they guard against escaping a *fixed* base dir (like RESULTS_DIR), not
|
|
119
|
+
# arbitrary filesystem browsing. This route only lists entries, never
|
|
120
|
+
# reads or serves file content.
|
|
121
|
+
#
|
|
122
|
+
# Params:
|
|
123
|
+
# :path - directory to list (defaults to the server's launch directory,
|
|
124
|
+
# Dir.pwd - same convention as Hlsv.config_path)
|
|
125
|
+
# :mode - 'directory' (default) or 'file'; in 'file' mode, files
|
|
126
|
+
# matching :ext are listed alongside subfolders
|
|
127
|
+
# :ext - extension filter for 'file' mode, e.g. '.xml'
|
|
128
|
+
get '/browse' do
|
|
92
129
|
content_type :json
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
130
|
+
|
|
131
|
+
requested = params['path'].to_s.strip
|
|
132
|
+
requested = Dir.pwd if requested.empty? # same convention as Hlsv.config_path in hlsv.rb
|
|
133
|
+
|
|
134
|
+
full_path = File.expand_path(requested)
|
|
135
|
+
|
|
136
|
+
halt 404, { error: 'Not found' }.to_json unless File.directory?(full_path)
|
|
137
|
+
halt 403, { error: 'Access denied' }.to_json unless File.readable?(full_path)
|
|
138
|
+
|
|
139
|
+
mode = params['mode'] == 'file' ? 'file' : 'directory'
|
|
140
|
+
ext_filter = params['ext'].to_s.strip.downcase
|
|
141
|
+
|
|
142
|
+
folders = []
|
|
143
|
+
files = []
|
|
144
|
+
|
|
145
|
+
Dir.children(full_path).sort_by(&:downcase).each do |name|
|
|
146
|
+
next if name.start_with?('.') # skip hidden files/folders
|
|
147
|
+
|
|
148
|
+
entry_path = File.join(full_path, name)
|
|
149
|
+
|
|
150
|
+
if File.directory?(entry_path)
|
|
151
|
+
folders << { name: name, path: entry_path }
|
|
152
|
+
else
|
|
153
|
+
# In 'directory' mode, files are listed unfiltered so the user can
|
|
154
|
+
# confirm they're in the right place; the ext filter only applies
|
|
155
|
+
# when actually selecting a file (mode == 'file').
|
|
156
|
+
next if mode == 'file' && !ext_filter.empty? && File.extname(name).downcase != ext_filter
|
|
157
|
+
|
|
158
|
+
files << { name: name, path: entry_path }
|
|
159
|
+
end
|
|
120
160
|
end
|
|
161
|
+
|
|
162
|
+
parent = File.dirname(full_path)
|
|
163
|
+
parent = nil if parent == full_path # already at filesystem root
|
|
164
|
+
|
|
165
|
+
{ path: full_path, parent: parent, folders: folders, files: files }.to_json
|
|
166
|
+
rescue Errno::EACCES
|
|
167
|
+
halt 403, { error: 'Access denied' }.to_json
|
|
121
168
|
end
|
|
122
169
|
|
|
123
170
|
# ---------------------------------------------------------------------------
|
|
124
|
-
# :section: ROUTES
|
|
171
|
+
# :section: ROUTES - Processing
|
|
125
172
|
# ---------------------------------------------------------------------------
|
|
126
173
|
|
|
127
|
-
# POST /
|
|
128
|
-
post '/
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
errors = validate_config(config)
|
|
133
|
-
|
|
134
|
-
if errors.any?
|
|
135
|
-
return {
|
|
136
|
-
success: false,
|
|
137
|
-
erreur: "Incomplete configuration",
|
|
138
|
-
details: errors
|
|
139
|
-
}.to_json
|
|
140
|
-
end
|
|
174
|
+
# POST /proceed - Validates config then runs the main processing script
|
|
175
|
+
post '/proceed' do
|
|
176
|
+
json_action do
|
|
177
|
+
config = ConfigManager.load
|
|
178
|
+
errors = ConfigManager.validate(config)
|
|
141
179
|
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
status 500
|
|
146
|
-
{ success: false, erreur: e.message }.to_json
|
|
180
|
+
halt 200, { success: false, error: "Incomplete configuration", details: errors }.to_json if errors.any?
|
|
181
|
+
|
|
182
|
+
{ result: AnalysisRunner.run(config) }
|
|
147
183
|
end
|
|
148
184
|
end
|
|
149
185
|
|
|
150
186
|
# ---------------------------------------------------------------------------
|
|
151
|
-
# :section: ROUTES
|
|
187
|
+
# :section: ROUTES - Results browsing
|
|
152
188
|
# ---------------------------------------------------------------------------
|
|
189
|
+
# GET /ping — Lightweight health check used by report.html to detect
|
|
190
|
+
# whether the Sinatra server backing it is reachable (vs. the file being
|
|
191
|
+
# opened offline, or served by an unrelated static file server)
|
|
192
|
+
get '/ping' do
|
|
193
|
+
content_type :json
|
|
194
|
+
{ success: true }.to_json
|
|
195
|
+
end
|
|
153
196
|
|
|
154
|
-
# GET /
|
|
155
|
-
get '/
|
|
197
|
+
# GET /results - Returns the hlsv_results/ directory tree as JSON
|
|
198
|
+
get '/results' do
|
|
156
199
|
content_type :json
|
|
157
|
-
results_dir = 'hlsv_results'
|
|
158
200
|
|
|
159
|
-
unless Dir.exist?(
|
|
160
|
-
return { success: true,
|
|
201
|
+
unless Dir.exist?(RESULTS_DIR)
|
|
202
|
+
return { success: true, tree: {} }.to_json
|
|
161
203
|
end
|
|
162
204
|
|
|
163
|
-
|
|
164
|
-
{ success: true, arborescence: arborescence }.to_json
|
|
205
|
+
{ success: true, tree: build_tree(RESULTS_DIR) }.to_json
|
|
165
206
|
end
|
|
166
207
|
|
|
167
|
-
# GET /
|
|
168
|
-
# Inline for: .html, .htm, .pdf, .png, .jpg, .jpeg, .gif, .svg, .txt, .css, .js
|
|
169
|
-
# Download for: all other extensions
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
208
|
+
# GET /download/* - Serves a result file (inline or as download)
|
|
209
|
+
# Inline by default for: .html, .htm, .pdf, .png, .jpg, .jpeg, .gif, .svg, .txt, .css, .js
|
|
210
|
+
# Download for: all other extensions, or any extension when ?download=1 is passed
|
|
211
|
+
# (used by the "⬇️ Download" button, as opposed to "👁️ Open" which omits it)
|
|
212
|
+
get '/download/*' do
|
|
213
|
+
file_path = resolve_within!(RESULTS_DIR, params['splat'].first)
|
|
214
|
+
halt 404, "File not found: #{params['splat'].first}" unless File.exist?(file_path)
|
|
173
215
|
|
|
174
|
-
|
|
175
|
-
|
|
216
|
+
extension = File.extname(file_path).downcase
|
|
217
|
+
force_download = params[:download] == '1'
|
|
176
218
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
if ['.html', '.htm', '.pdf', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.txt'].include?(extension)
|
|
219
|
+
if !force_download && ['.html', '.htm', '.pdf', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.txt'].include?(extension)
|
|
180
220
|
send_file file_path, filename: File.basename(file_path), disposition: 'inline'
|
|
181
|
-
elsif ['.css', '.js'].include?(extension)
|
|
221
|
+
elsif !force_download && ['.css', '.js'].include?(extension)
|
|
182
222
|
content_type extension == '.css' ? 'text/css' : 'application/javascript'
|
|
183
223
|
send_file file_path, disposition: 'inline'
|
|
184
224
|
else
|
|
@@ -186,12 +226,10 @@ class WebApp < Sinatra::Base
|
|
|
186
226
|
end
|
|
187
227
|
end
|
|
188
228
|
|
|
189
|
-
# GET /
|
|
190
|
-
get '/
|
|
229
|
+
# GET /download_zip_dir/* - Packages a result folder as a ZIP archive
|
|
230
|
+
get '/download_zip_dir/*' do
|
|
191
231
|
relative_folder = params['splat'].first
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
folder_path = File.join('hlsv_results', relative_folder)
|
|
232
|
+
folder_path = resolve_within!(RESULTS_DIR, relative_folder)
|
|
195
233
|
halt 404, "Folder not found" unless Dir.exist?(folder_path)
|
|
196
234
|
|
|
197
235
|
# Build ZIP in memory, preserving relative paths inside the folder
|
|
@@ -201,254 +239,110 @@ class WebApp < Sinatra::Base
|
|
|
201
239
|
|
|
202
240
|
relative_path = file_path.sub("#{folder_path}/", '')
|
|
203
241
|
zip.put_next_entry(relative_path)
|
|
204
|
-
zip.write(File.
|
|
242
|
+
zip.write(File.binread(file_path)) # binread: preserves binary files (pdf, png, ...)
|
|
205
243
|
end
|
|
206
244
|
end
|
|
207
245
|
|
|
208
246
|
zip_data.rewind
|
|
209
|
-
content_type
|
|
210
|
-
|
|
211
|
-
attachment "#{Time.now.strftime('%Y-%m-%d')}-#{folder_name}.zip"
|
|
247
|
+
content_type 'application/zip'
|
|
248
|
+
attachment "#{Time.now.strftime('%Y-%m-%d')}-#{File.basename(relative_folder)}.zip"
|
|
212
249
|
zip_data.read
|
|
213
250
|
end
|
|
214
251
|
|
|
215
252
|
# ---------------------------------------------------------------------------
|
|
216
|
-
# :section: ROUTES
|
|
253
|
+
# :section: ROUTES - On-demand report generation (html / docx / excel)
|
|
217
254
|
# ---------------------------------------------------------------------------
|
|
218
|
-
|
|
219
|
-
#
|
|
220
|
-
#
|
|
221
|
-
#
|
|
222
|
-
#
|
|
223
|
-
#
|
|
224
|
-
#
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
footer_format = workbook.add_format(font_size: 9, font_color: "#6c757d", align: "center")
|
|
241
|
-
header_format = workbook.add_format(bold: true, bg_color: "#c0c0c0", pattern: 1, border: 1)
|
|
242
|
-
even_format = workbook.add_format(pattern: 1, border: 1)
|
|
243
|
-
odd_format = workbook.add_format(bg_color: "#E3F2FD", pattern: 1, border: 1)
|
|
244
|
-
|
|
245
|
-
# --- Sheet 1: README ---
|
|
246
|
-
readme_sheet = workbook.add_worksheet("README")
|
|
247
|
-
readme_sheet.set_column(0, 0, 80)
|
|
248
|
-
readme_sheet.set_column(1, 1, 20)
|
|
249
|
-
|
|
250
|
-
current_row = 0
|
|
251
|
-
readme_sheet.write_string(current_row, 0, "Duplicates Analysis Report - #{sheet_name}", title_format)
|
|
252
|
-
current_row += 2
|
|
253
|
-
|
|
254
|
-
readme_sheet.write_string(current_row, 0, "About This Report", heading_format)
|
|
255
|
-
current_row += 1
|
|
256
|
-
|
|
257
|
-
[
|
|
258
|
-
"This Excel file displays the detected duplicates, grouped according to the last key tested.",
|
|
259
|
-
"The duplicate groups are represented by a number in the 'No' column.",
|
|
260
|
-
"Alternating colors (white and light blue) are used to distinguish them visually.",
|
|
261
|
-
"All variables present in the dataset are displayed."
|
|
262
|
-
].each { |line| readme_sheet.write_string(current_row, 0, line, text_format); current_row += 1 }
|
|
263
|
-
current_row += 1
|
|
264
|
-
|
|
265
|
-
readme_sheet.write_string(current_row, 0, "Sheet Information", heading_format)
|
|
266
|
-
current_row += 1
|
|
267
|
-
|
|
268
|
-
[
|
|
269
|
-
"• Dataset: #{sheet_name}",
|
|
270
|
-
"• Duplicate Type: #{type_dup == 'data' ? 'Duplicates in dataset' : 'Duplicates in define.xml'}",
|
|
271
|
-
"• Total Records: #{csv.size}",
|
|
272
|
-
"• Number of Variables: #{csv.headers.size}",
|
|
273
|
-
"• Generated: #{Time.now.strftime('%Y-%m-%d at %H:%M')}",
|
|
274
|
-
"• Last key tested: #{last_valid_key.join(' ')}"
|
|
275
|
-
].each { |line| readme_sheet.write_string(current_row, 0, line, text_format); current_row += 1 }
|
|
276
|
-
current_row += 1
|
|
277
|
-
|
|
278
|
-
readme_sheet.write_string(current_row, 0, "Next Steps", heading_format)
|
|
279
|
-
current_row += 1
|
|
280
|
-
|
|
281
|
-
[
|
|
282
|
-
"→ Click on the '#{sheet_name}' tab at the bottom to view the duplicate records.",
|
|
283
|
-
"→ Identify variables to add in the configuration form to remove this duplicate.",
|
|
284
|
-
"→ Highlight issues with data cleaning."
|
|
285
|
-
].each { |line| readme_sheet.write_string(current_row, 0, line, text_format); current_row += 1 }
|
|
286
|
-
current_row += 1
|
|
287
|
-
|
|
288
|
-
readme_sheet.write_string(current_row, 0,
|
|
289
|
-
"© #{Time.now.year} AdClin. All rights reserved. | Licensed under AGPL v3",
|
|
290
|
-
footer_format)
|
|
291
|
-
|
|
292
|
-
# --- Sheet 2: Duplicate data ---
|
|
293
|
-
data_sheet = workbook.add_worksheet(sheet_name)
|
|
294
|
-
col_widths = csv.headers.map { |h| h.to_s.length }
|
|
295
|
-
|
|
296
|
-
# Write frozen header row
|
|
297
|
-
csv.headers.each_with_index { |h, col| data_sheet.write_string(0, col, h, header_format) }
|
|
298
|
-
data_sheet.freeze_panes(1, 5)
|
|
299
|
-
|
|
300
|
-
# Write data rows with alternating colors based on the 'No' column (col 0)
|
|
301
|
-
csv.each_with_index do |row, row_index|
|
|
302
|
-
fmt = row[0].to_i.even? ? even_format : odd_format
|
|
303
|
-
row.fields.each_with_index do |value, col|
|
|
304
|
-
value_str = value.to_s
|
|
305
|
-
data_sheet.write_string(row_index + 1, col, value_str, fmt)
|
|
306
|
-
col_widths[col] = [col_widths[col], value_str.length].max
|
|
255
|
+
#
|
|
256
|
+
# These routes drive `sv.report`, the engine kept in memory by
|
|
257
|
+
# AnalysisRegistry after '/proceed' completed. Nothing is written to disk
|
|
258
|
+
# until one of these routes is actually hit - reports are generated once,
|
|
259
|
+
# on click, not automatically after the analysis.
|
|
260
|
+
#
|
|
261
|
+
# :study is the *display* study name (not the "hlsv_results/" prefixed
|
|
262
|
+
# internal one) - same value as the `study_name` field returned by
|
|
263
|
+
# POST /proceed.
|
|
264
|
+
|
|
265
|
+
# GET /report/:study/html - Serves the already-generated report if present
|
|
266
|
+
# on disk (works even without the analysis in memory, e.g. after a server
|
|
267
|
+
# restart); regenerates it on demand otherwise.
|
|
268
|
+
get '/report/:study/html' do
|
|
269
|
+
path = report_path_for(params[:study], 'html')
|
|
270
|
+
|
|
271
|
+
if File.exist?(path)
|
|
272
|
+
send_file path, disposition: 'inline'
|
|
273
|
+
else
|
|
274
|
+
with_analysis(params[:study]) do |sv|
|
|
275
|
+
sv.report.html
|
|
276
|
+
send_file "#{sv.report.report_name}.html", disposition: 'inline'
|
|
307
277
|
end
|
|
308
278
|
end
|
|
309
|
-
|
|
310
|
-
col_widths.each_with_index { |width, col| data_sheet.set_column(col, col, width + 2) }
|
|
311
|
-
|
|
312
|
-
# Send file
|
|
313
|
-
type = type_dup == 'data' ? 'duplicate_in_dataset' : 'duplicate_in_define'
|
|
314
|
-
file_name = "#{Time.now.strftime('%Y-%m-%d')}-#{type}.xlsx"
|
|
315
|
-
content_type 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
316
|
-
attachment file_name
|
|
317
|
-
workbook.read_string
|
|
318
279
|
end
|
|
319
280
|
|
|
320
|
-
# GET /
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
get '/telecharger_dossier_excel/*' do
|
|
324
|
-
relative_folder = params['splat'].first
|
|
325
|
-
halt 403, "Access denied" if relative_folder.include?('..') || relative_folder.start_with?('/')
|
|
326
|
-
|
|
327
|
-
folder_path = File.join('hlsv_results', relative_folder)
|
|
328
|
-
halt 404, "Folder not found" unless Dir.exist?(folder_path)
|
|
329
|
-
|
|
330
|
-
csv_files = Dir.glob(File.join(folder_path, '*.csv')).sort
|
|
331
|
-
halt 404, "No CSV files found in this folder" if csv_files.empty?
|
|
332
|
-
|
|
333
|
-
workbook = FastExcel.open(constant_memory: true)
|
|
334
|
-
folder_name = File.dirname(relative_folder)
|
|
335
|
-
|
|
336
|
-
# --- Shared formats ---
|
|
337
|
-
title_format = workbook.add_format(bold: true, font_size: 16, font_color: "#2c3e50", align: "left")
|
|
338
|
-
heading_format = workbook.add_format(bold: true, font_size: 12, font_color: "#2c3e50", bg_color: "#e8f4f8", pattern: 1, border: 1)
|
|
339
|
-
text_format = workbook.add_format(font_size: 11, align: "left", text_wrap: true)
|
|
340
|
-
footer_format = workbook.add_format(font_size: 9, font_color: "#6c757d", align: "center")
|
|
341
|
-
header_format = workbook.add_format(bold: true, bg_color: "#c0c0c0", pattern: 1, border: 1)
|
|
342
|
-
even_format = workbook.add_format(pattern: 1, border: 1)
|
|
343
|
-
odd_format = workbook.add_format(bg_color: "#E3F2FD", pattern: 1, border: 1)
|
|
344
|
-
|
|
345
|
-
# --- Sheet 1: README ---
|
|
346
|
-
readme_sheet = workbook.add_worksheet("README")
|
|
347
|
-
readme_sheet.set_column(0, 0, 80)
|
|
348
|
-
readme_sheet.set_column(1, 1, 20)
|
|
349
|
-
|
|
350
|
-
current_row = 0
|
|
351
|
-
readme_sheet.write_string(current_row, 0, "Duplicates Analysis Report - #{folder_name}", title_format)
|
|
352
|
-
current_row += 2
|
|
353
|
-
|
|
354
|
-
readme_sheet.write_string(current_row, 0, "About This Report", heading_format)
|
|
355
|
-
current_row += 1
|
|
356
|
-
|
|
357
|
-
[
|
|
358
|
-
"This Excel file displays the detected duplicates, grouped according to the last key tested.",
|
|
359
|
-
"The duplicate groups are represented by a number in the 'No' column.",
|
|
360
|
-
"Alternating colors (white and light blue) are used to distinguish them visually.",
|
|
361
|
-
"All variables present in the dataset are displayed."
|
|
362
|
-
].each { |line| readme_sheet.write_string(current_row, 0, line, text_format); current_row += 1 }
|
|
363
|
-
current_row += 1
|
|
364
|
-
|
|
365
|
-
readme_sheet.write_string(current_row, 0, "Workbook Information", heading_format)
|
|
366
|
-
current_row += 1
|
|
367
|
-
|
|
368
|
-
[
|
|
369
|
-
"• Folder: #{folder_name}",
|
|
370
|
-
"• Number of Datasets: #{csv_files.size}",
|
|
371
|
-
"• Generated: #{Time.now.strftime('%Y-%m-%d at %H:%M')}"
|
|
372
|
-
].each { |line| readme_sheet.write_string(current_row, 0, line, text_format); current_row += 1 }
|
|
373
|
-
current_row += 1
|
|
374
|
-
|
|
375
|
-
readme_sheet.write_string(current_row, 0, "Next Steps", heading_format)
|
|
376
|
-
current_row += 1
|
|
377
|
-
|
|
378
|
-
[
|
|
379
|
-
"→ Each tab corresponds to one CSV file from the folder.",
|
|
380
|
-
"→ Review duplicate records and identify variables to add in the configuration.",
|
|
381
|
-
"→ Highlight issues with data cleaning."
|
|
382
|
-
].each { |line| readme_sheet.write_string(current_row, 0, line, text_format); current_row += 1 }
|
|
383
|
-
current_row += 1
|
|
384
|
-
|
|
385
|
-
readme_sheet.write_string(current_row, 0,
|
|
386
|
-
"© #{Time.now.year} AdClin. All rights reserved. | Licensed under AGPL v3",
|
|
387
|
-
footer_format)
|
|
388
|
-
|
|
389
|
-
# --- Data sheets: one per CSV file ---
|
|
390
|
-
csv_files.each do |csv_file|
|
|
391
|
-
sheet_name = File.basename(csv_file, '.csv')
|
|
392
|
-
sheet_name = sheet_name[0..30] if sheet_name.length > 31 # Excel sheet name limit: 31 chars
|
|
393
|
-
|
|
394
|
-
csv_data = CSV.read(csv_file, headers: true)
|
|
395
|
-
next if csv_data.empty?
|
|
396
|
-
|
|
397
|
-
data_sheet = workbook.add_worksheet(sheet_name)
|
|
398
|
-
col_widths = csv_data.headers.map { |h| h.to_s.length }
|
|
399
|
-
|
|
400
|
-
# Write frozen header row
|
|
401
|
-
csv_data.headers.each_with_index { |header, col| data_sheet.write_string(0, col, header.to_s, header_format) }
|
|
402
|
-
data_sheet.freeze_panes(1, 5)
|
|
403
|
-
|
|
404
|
-
# Write data rows with alternating colors based on the 'No' column (col 0)
|
|
405
|
-
csv_data.each_with_index do |row, row_index|
|
|
406
|
-
fmt = row[0].to_i.even? ? even_format : odd_format
|
|
407
|
-
row.fields.each_with_index do |value, col|
|
|
408
|
-
value_str = value.to_s
|
|
409
|
-
data_sheet.write_string(row_index + 1, col, value_str, fmt)
|
|
410
|
-
col_widths[col] = [col_widths[col], value_str.length].max
|
|
411
|
-
end
|
|
412
|
-
end
|
|
281
|
+
# GET /report/:study/docx
|
|
282
|
+
get '/report/:study/docx' do
|
|
283
|
+
path = report_path_for(params[:study], 'docx')
|
|
413
284
|
|
|
414
|
-
|
|
285
|
+
if File.exist?(path)
|
|
286
|
+
send_file path, filename: "#{params[:study]}.docx"
|
|
287
|
+
else
|
|
288
|
+
with_analysis(params[:study]) do |sv|
|
|
289
|
+
sv.report.docx
|
|
290
|
+
send_file "#{sv.report.report_name}.docx", filename: "#{params[:study]}.docx"
|
|
291
|
+
end
|
|
415
292
|
end
|
|
416
|
-
|
|
417
|
-
# Send file
|
|
418
|
-
content_type 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
419
|
-
attachment "#{Time.now.strftime('%Y-%m-%d')}_#{folder_name}.xlsx"
|
|
420
|
-
workbook.read_string
|
|
421
293
|
end
|
|
422
294
|
|
|
423
|
-
#
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
# GET /telecharger_html_word/* — Converts an HTML result file to a .docx Word document
|
|
428
|
-
# Returns JSON with success status and generated filename
|
|
429
|
-
get '/telecharger_html_word/*' do
|
|
430
|
-
content_type :json
|
|
431
|
-
chemin_relatif = params['splat'].first
|
|
295
|
+
# GET /report/:study/excel - whole-folder export
|
|
296
|
+
get '/report/:study/excel' do
|
|
297
|
+
path = report_path_for(params[:study], 'xlsx')
|
|
432
298
|
|
|
433
|
-
|
|
434
|
-
|
|
299
|
+
if File.exist?(path)
|
|
300
|
+
send_file path, filename: "#{params[:study]}.xlsx"
|
|
301
|
+
else
|
|
302
|
+
with_analysis(params[:study]) do |sv|
|
|
303
|
+
sv.report.excel
|
|
304
|
+
halt 404, "No duplicates found - nothing to export" unless File.exist?("#{sv.report.report_name}.xlsx")
|
|
305
|
+
send_file "#{sv.report.report_name}.xlsx", filename: "#{params[:study]}.xlsx"
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
end
|
|
435
309
|
|
|
436
|
-
|
|
310
|
+
# GET /report/:study/excel/:dataset - per-dataset export
|
|
311
|
+
# obsolete keep in case of
|
|
312
|
+
get '/report/:study/excel/:dataset' do
|
|
313
|
+
path = "#{RESULTS_DIR}/#{params[:study]}/#{params[:dataset]}.xlsx"
|
|
437
314
|
|
|
438
|
-
|
|
439
|
-
|
|
315
|
+
if File.exist?(path)
|
|
316
|
+
send_file path, filename: "#{params[:dataset]}.xlsx"
|
|
317
|
+
else
|
|
318
|
+
with_analysis(params[:study]) do |sv|
|
|
319
|
+
sv.report.excel(dataset: params[:dataset])
|
|
320
|
+
halt 404, "No duplicates found for #{params[:dataset]} - nothing to export" unless File.exist?(path)
|
|
321
|
+
send_file path, filename: "#{params[:dataset]}.xlsx"
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
end
|
|
440
325
|
|
|
441
|
-
|
|
326
|
+
# GET /export_csv_excel - Exports a single duplicate CSV (the one shown on
|
|
327
|
+
# /csv_view) as a one-sheet Excel workbook. Works fully offline: no
|
|
328
|
+
# AnalysisRegistry lookup, since /csv_view's static HTML link already
|
|
329
|
+
# carries every parameter needed (file, dataset, tested key).
|
|
330
|
+
get '/export_csv_excel' do
|
|
331
|
+
study = params[:study]
|
|
332
|
+
dataset = params[:dataset]
|
|
333
|
+
key = params[:last_valid_key]
|
|
334
|
+
file = safe_relative_path!(params[:file])
|
|
442
335
|
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
puts " -> #{blocks.size} blocks extracted"
|
|
336
|
+
halt 400, "Missing required parameters" unless study && dataset
|
|
337
|
+
halt 404, "File not found" unless File.exist?(file)
|
|
446
338
|
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
339
|
+
workbook = Hlsv::SdtmValidation::Report.build_single_csv_workbook(
|
|
340
|
+
study_name: study, ds_name: dataset, csv_path: file, key: key
|
|
341
|
+
)
|
|
450
342
|
|
|
451
|
-
|
|
343
|
+
content_type 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
344
|
+
attachment "#{dataset}.xlsx"
|
|
345
|
+
workbook.read_string
|
|
452
346
|
end
|
|
453
347
|
|
|
454
348
|
# ---------------------------------------------------------------------------
|
|
@@ -456,11 +350,39 @@ class WebApp < Sinatra::Base
|
|
|
456
350
|
# ---------------------------------------------------------------------------
|
|
457
351
|
|
|
458
352
|
helpers do
|
|
353
|
+
include Hlsv::PathGuard
|
|
354
|
+
include Hlsv::UrlHelper
|
|
355
|
+
|
|
356
|
+
# Runs the block, wraps its return hash into {success: true, **hash}.to_json,
|
|
357
|
+
# and turns any raised error into {success: false, error: message} with a 500.
|
|
358
|
+
def json_action
|
|
359
|
+
content_type :json
|
|
360
|
+
{ success: true, **yield }.to_json
|
|
361
|
+
rescue => e
|
|
362
|
+
status 500
|
|
363
|
+
{ success: false, error: e.message }.to_json
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
# Fetches the in-memory analysis for `study` and yields it to the block.
|
|
367
|
+
# Turns a missing/unknown study into a clean 404 instead of a stack trace,
|
|
368
|
+
# since it's a legitimate case (server restarted, or study never analyzed).
|
|
369
|
+
def with_analysis(study)
|
|
370
|
+
sv = AnalysisRegistry.fetch!(study)
|
|
371
|
+
yield sv
|
|
372
|
+
rescue Hlsv::Error => e
|
|
373
|
+
halt 404, e.message
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
# Path of a report file for a given (display) study name, independent
|
|
377
|
+
# of whether the analysis is currently in memory
|
|
378
|
+
def report_path_for(study, ext)
|
|
379
|
+
"#{Hlsv::SdtmValidation::Report.report_name_for("#{RESULTS_DIR}/#{study}")}.#{ext}"
|
|
380
|
+
end
|
|
459
381
|
|
|
460
382
|
# Recursively builds a directory tree hash for a given base path.
|
|
461
|
-
# Returns: {
|
|
383
|
+
# Returns: { files: [...], folders: { name => subtree, ... } }
|
|
462
384
|
def build_tree(base_path, relative_path = '')
|
|
463
|
-
tree = {
|
|
385
|
+
tree = { files: [], folders: {} }
|
|
464
386
|
full_path = relative_path.empty? ? base_path : File.join(base_path, relative_path)
|
|
465
387
|
|
|
466
388
|
return tree unless Dir.exist?(full_path)
|
|
@@ -472,97 +394,21 @@ class WebApp < Sinatra::Base
|
|
|
472
394
|
relative_entry_path = relative_path.empty? ? entry : File.join(relative_path, entry)
|
|
473
395
|
|
|
474
396
|
if File.directory?(entry_path)
|
|
475
|
-
tree[:
|
|
397
|
+
tree[:folders][entry] = build_tree(base_path, relative_entry_path)
|
|
476
398
|
else
|
|
477
|
-
tree[:
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
399
|
+
tree[:files] << {
|
|
400
|
+
name: entry,
|
|
401
|
+
path: relative_entry_path,
|
|
402
|
+
size: File.size(entry_path),
|
|
481
403
|
date: File.mtime(entry_path).strftime('%Y-%m-%d %H:%M:%S'),
|
|
482
404
|
extension: File.extname(entry)
|
|
483
405
|
}
|
|
484
406
|
end
|
|
485
407
|
end
|
|
486
408
|
|
|
487
|
-
tree[:
|
|
409
|
+
tree[:files].sort_by! { |f| f[:name] }
|
|
488
410
|
tree
|
|
489
411
|
end
|
|
490
|
-
|
|
491
|
-
# Loads and returns config.yaml as a Hash. Halts with 500 if file is missing.
|
|
492
|
-
def load_config
|
|
493
|
-
if File.exist?(Hlsv.config_path)
|
|
494
|
-
YAML.load_file(Hlsv.config_path) || {}
|
|
495
|
-
else
|
|
496
|
-
halt 500, "File config.yaml not found"
|
|
497
|
-
end
|
|
498
|
-
end
|
|
499
|
-
|
|
500
|
-
# Merges config_params into the existing config.yaml, for editable fields only.
|
|
501
|
-
# output_type is always forced to 'csv'.
|
|
502
|
-
def save_config(config_params)
|
|
503
|
-
current_config = File.exist?(Hlsv.config_path) ? (YAML.load_file(Hlsv.config_path) || {}) : {}
|
|
504
|
-
|
|
505
|
-
editable_fields = %w(
|
|
506
|
-
study_name output_directory data_directory define_path excluded_ds
|
|
507
|
-
event_key intervention_key finding_key finding_about_key
|
|
508
|
-
ds_key relrec_key CO_key TA_key TE_key TI_key TS_key TV_key
|
|
509
|
-
)
|
|
510
|
-
|
|
511
|
-
editable_fields.each do |field|
|
|
512
|
-
current_config[field] = config_params[field] if config_params.key?(field)
|
|
513
|
-
end
|
|
514
|
-
|
|
515
|
-
current_config['output_type'] = 'csv'
|
|
516
|
-
File.write(Hlsv.config_path, current_config.to_yaml)
|
|
517
|
-
end
|
|
518
|
-
|
|
519
|
-
# Validates the configuration hash.
|
|
520
|
-
# Checks all required fields are present and validates filesystem paths.
|
|
521
|
-
# Returns an array of error messages (empty if config is valid).
|
|
522
|
-
def validate_config(config)
|
|
523
|
-
errors = []
|
|
524
|
-
|
|
525
|
-
required_fields = {
|
|
526
|
-
'study_name' => 'Study name',
|
|
527
|
-
'output_directory' => 'Output directory',
|
|
528
|
-
'data_directory' => 'Data directory',
|
|
529
|
-
'define_path' => 'Define.xml path',
|
|
530
|
-
'event_key' => 'Event key',
|
|
531
|
-
'intervention_key' => 'Intervention key',
|
|
532
|
-
'finding_key' => 'Finding key',
|
|
533
|
-
'finding_about_key' => 'Finding about key',
|
|
534
|
-
'ds_key' => 'DS key',
|
|
535
|
-
'relrec_key' => 'RELREC key',
|
|
536
|
-
'CO_key' => 'CO key',
|
|
537
|
-
'TA_key' => 'TA key',
|
|
538
|
-
'TE_key' => 'TE key',
|
|
539
|
-
'TI_key' => 'TI key',
|
|
540
|
-
'TS_key' => 'TS key',
|
|
541
|
-
'TV_key' => 'TV key'
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
# Check all required fields are filled
|
|
545
|
-
required_fields.each do |key, name|
|
|
546
|
-
value = config[key]
|
|
547
|
-
errors << "#{name} is empty" if value.nil? || value.to_s.strip.empty?
|
|
548
|
-
end
|
|
549
|
-
|
|
550
|
-
# Check data_directory exists and contains at least one .xpt file
|
|
551
|
-
if config['data_directory'] && !config['data_directory'].to_s.strip.empty?
|
|
552
|
-
dir = config['data_directory'].gsub('\\', '/')
|
|
553
|
-
errors << "Data directory does not exist: #{dir}" unless Dir.exist?(dir)
|
|
554
|
-
errors << "Directory is empty, no .xpt files detected" if Dir["#{dir}/*"].none? { |f| File.extname(f) == '.xpt' }
|
|
555
|
-
end
|
|
556
|
-
|
|
557
|
-
# Check define_path exists (skip if value is '-', meaning no define file)
|
|
558
|
-
if config['define_path'] && config['define_path'] != '-'
|
|
559
|
-
errors << "Invalid path: #{config['define_path']}" unless File.exist?(config['define_path'])
|
|
560
|
-
end
|
|
561
|
-
|
|
562
|
-
config['output_type'] = 'csv'
|
|
563
|
-
errors
|
|
564
|
-
end
|
|
565
|
-
|
|
566
412
|
end # helpers
|
|
567
413
|
|
|
568
414
|
end # class WebApp
|