oddb2xml 2.7.1 → 2.7.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +1 -2
- data/.standard.yml +2 -0
- data/Gemfile +3 -3
- data/History.txt +24 -0
- data/README.md +3 -3
- data/Rakefile +24 -23
- data/bin/check_artikelstamm +11 -11
- data/bin/compare_v5 +23 -23
- data/bin/oddb2xml +14 -13
- data/lib/oddb2xml/builder.rb +1070 -1038
- data/lib/oddb2xml/calc.rb +232 -233
- data/lib/oddb2xml/chapter_70_hack.rb +38 -32
- data/lib/oddb2xml/cli.rb +252 -236
- data/lib/oddb2xml/compare.rb +70 -59
- data/lib/oddb2xml/compositions_syntax.rb +451 -430
- data/lib/oddb2xml/compressor.rb +20 -20
- data/lib/oddb2xml/downloader.rb +157 -129
- data/lib/oddb2xml/extractor.rb +295 -295
- data/lib/oddb2xml/options.rb +34 -35
- data/lib/oddb2xml/parslet_compositions.rb +265 -269
- data/lib/oddb2xml/semantic_check.rb +39 -33
- data/lib/oddb2xml/util.rb +163 -163
- data/lib/oddb2xml/version.rb +1 -1
- data/lib/oddb2xml/xml_definitions.rb +32 -33
- data/lib/oddb2xml.rb +1 -1
- data/oddb2xml.gemspec +34 -34
- data/shell.nix +17 -0
- data/spec/artikelstamm_spec.rb +111 -110
- data/spec/builder_spec.rb +490 -505
- data/spec/calc_spec.rb +552 -593
- data/spec/check_artikelstamm_spec.rb +26 -26
- data/spec/cli_spec.rb +173 -174
- data/spec/compare_spec.rb +9 -11
- data/spec/composition_syntax_spec.rb +390 -409
- data/spec/compressor_spec.rb +48 -48
- data/spec/data/transfer.dat +1 -0
- data/spec/data_helper.rb +47 -49
- data/spec/downloader_spec.rb +251 -260
- data/spec/extractor_spec.rb +171 -159
- data/spec/fixtures/vcr_cassettes/oddb2xml.json +1 -1
- data/spec/galenic_spec.rb +233 -256
- data/spec/options_spec.rb +116 -119
- data/spec/parslet_spec.rb +896 -863
- data/spec/spec_helper.rb +153 -153
- data/test_options.rb +39 -42
- data/tools/win_fetch_cacerts.rb +2 -3
- metadata +42 -12
data/spec/downloader_spec.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
|
-
|
2
|
-
require 'spec_helper'
|
1
|
+
require "spec_helper"
|
3
2
|
require "rexml/document"
|
4
3
|
|
5
4
|
VCR.eject_cassette # we use insert/eject around each example
|
6
5
|
|
7
6
|
# not used but, as I still don't know how to generate
|
8
|
-
def filter_aips_xml(filename=
|
9
|
-
puts "File #{filename} exists? #{File.
|
10
|
-
|
7
|
+
def filter_aips_xml(filename = "AipsDownload_ng.xml", ids_to_keep = [55558, 61848])
|
8
|
+
puts "File #{filename} exists? #{File.exist?(filename)}"
|
9
|
+
if $VERBOSE
|
10
|
+
puts %(<?xml version="1.0" encoding="utf-8"?>
|
11
11
|
<medicalInformations>
|
12
12
|
<medicalInformation type="fi" version="5" lang="de" safetyRelevant="false" informationUpdate="07.2008">
|
13
13
|
<title>Zyvoxid®</title>
|
@@ -16,37 +16,36 @@ def filter_aips_xml(filename='AipsDownload_ng.xml', ids_to_keep = [55558, 61848]
|
|
16
16
|
<substances>Linezolid</substances>
|
17
17
|
<authNrs>55558, 55559, 55560</authNrs>
|
18
18
|
)
|
19
|
+
end
|
19
20
|
@xml = IO.read(filename)
|
20
|
-
ausgabe = File.open(
|
21
|
+
ausgabe = File.open("tst.out", "w+")
|
21
22
|
data = {}
|
22
|
-
result = MedicalInformationsContent.parse(@xml.sub(
|
23
|
+
result = MedicalInformationsContent.parse(@xml.sub(STRIP_FOR_SAX_MACHINE, ""), lazy: true)
|
23
24
|
result.medicalInformation.each do |pac|
|
24
25
|
lang = pac.lang.to_s
|
25
|
-
next unless
|
26
|
+
next unless /de|fr/.match?(lang)
|
26
27
|
item = {}
|
27
|
-
|
28
|
-
pac.authNrs.split(/[, ]+/).each{
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
break
|
28
|
+
keep_it = false
|
29
|
+
pac.authNrs.split(/[, ]+/).each { |id|
|
30
|
+
if ids_to_keep.index(id.to_i)
|
31
|
+
data[[lang, id.to_i]] = pac
|
32
|
+
keep_it = true
|
33
|
+
ausgabe.puts
|
34
|
+
break
|
35
35
|
end
|
36
36
|
}
|
37
|
-
html = Nokogiri::HTML.fragment(pac.content.force_encoding(
|
37
|
+
html = Nokogiri::HTML.fragment(pac.content.force_encoding("UTF-8"))
|
38
38
|
item[:paragraph] = html
|
39
|
-
numbers =
|
39
|
+
numbers = /(\d{5})[,\s]*(\d{5})?|(\d{5})[,\s]*(\d{5})?[,\s]*(\d{5})?/.match(html)
|
40
40
|
if numbers
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
puts "Must keep #{keepIt} #{pac.authNrs}"
|
41
|
+
[$1, $2, $3].compact.each { |id|
|
42
|
+
if ids_to_keep.index(id.to_i)
|
43
|
+
data[[lang, id.to_i]] = pac
|
44
|
+
keep_it = true
|
45
|
+
break
|
46
|
+
end
|
47
|
+
}
|
48
|
+
puts "Must keep #{keep_it} #{pac.authNrs}"
|
50
49
|
end
|
51
50
|
end
|
52
51
|
puts data.size
|
@@ -54,43 +53,32 @@ def filter_aips_xml(filename='AipsDownload_ng.xml', ids_to_keep = [55558, 61848]
|
|
54
53
|
end
|
55
54
|
|
56
55
|
XML_VERSION_1_0 = /xml\sversion=["']1.0["']/
|
57
|
-
PREP_XML =
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
expect {
|
62
|
-
Array.new(3).map do
|
63
|
-
Thread.new do
|
64
|
-
expect(@downloader.send(:retrievable?)).to be(true)
|
65
|
-
end
|
66
|
-
end.map(&:join)
|
67
|
-
}.to change {
|
68
|
-
@downloader.instance_variable_get(:@retry_times)
|
69
|
-
}.from(3).to(0)
|
70
|
-
end if false # as vcr does not support threads for the moment
|
56
|
+
PREP_XML = "Preparations.xml"
|
57
|
+
|
58
|
+
shared_examples_for "any downloader" do
|
59
|
+
# nothing at the moment
|
71
60
|
end
|
72
61
|
|
73
62
|
def common_before
|
74
|
-
@
|
63
|
+
@saved_dir = Dir.pwd
|
75
64
|
cleanup_directories_before_run
|
76
|
-
FileUtils.makedirs(Oddb2xml::
|
77
|
-
Dir.chdir(Oddb2xml::
|
65
|
+
FileUtils.makedirs(Oddb2xml::WORK_DIR)
|
66
|
+
Dir.chdir(Oddb2xml::WORK_DIR)
|
78
67
|
WebMock.enable!
|
79
|
-
{
|
80
|
-
}.each do |url, file|
|
68
|
+
{"https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/interactions_de_utf8.csv" => "epha_interactions.csv"}.each do |url, file|
|
81
69
|
inhalt = File.read(File.join(Oddb2xml::SpecData, file))
|
82
|
-
stub_request(:get,url).to_return(body: inhalt)
|
70
|
+
stub_request(:get, url).to_return(body: inhalt)
|
83
71
|
end
|
84
72
|
end
|
85
73
|
|
86
74
|
def common_after
|
87
|
-
Dir.chdir(@
|
75
|
+
Dir.chdir(@saved_dir) if @saved_dir && File.directory?(@saved_dir)
|
88
76
|
VCR.eject_cassette
|
89
|
-
vcr_file = File.expand_path(File.join(Oddb2xml::SpecData,
|
90
|
-
puts "Pretty-printing #{vcr_file} exists? #{File.
|
91
|
-
vcr_file_new = vcr_file.sub(
|
77
|
+
vcr_file = File.expand_path(File.join(Oddb2xml::SpecData, "..", "fixtures", "vcr_cassettes", "oddb2xml.json"))
|
78
|
+
puts "Pretty-printing #{vcr_file} exists? #{File.exist?(vcr_file)}" if $VERBOSE
|
79
|
+
vcr_file_new = vcr_file.sub(".json", ".new")
|
92
80
|
cmd = "cat #{vcr_file} | python -mjson.tool > #{vcr_file_new}"
|
93
|
-
|
81
|
+
system(cmd)
|
94
82
|
FileUtils.mv(vcr_file_new, vcr_file)
|
95
83
|
end
|
96
84
|
|
@@ -106,87 +94,85 @@ def zip_files(zipfile_name, input_filenames)
|
|
106
94
|
end
|
107
95
|
|
108
96
|
# Unzips into a specific directory
|
109
|
-
def unzip_files(zipfile_name, directory=Dir.pwd)
|
110
|
-
|
97
|
+
def unzip_files(zipfile_name, directory = Dir.pwd)
|
98
|
+
saved_dir = Dir.pwd
|
111
99
|
FileUtils.makedirs(directory)
|
112
100
|
Dir.chdir(directory)
|
113
101
|
Zip::File.open(zipfile_name) do |zip_file|
|
114
102
|
# Handle entries one by one
|
115
103
|
zip_file.each do |entry|
|
116
104
|
# Extract to file/directory/symlink
|
117
|
-
puts "downloader_spec.rb: Extracting #{entry.name} exists? #{File.
|
118
|
-
FileUtils.rm_f(entry.name, :
|
105
|
+
puts "downloader_spec.rb: Extracting #{entry.name} exists? #{File.exist?(entry.name)} into #{directory}"
|
106
|
+
FileUtils.rm_f(entry.name, verbose: true) if File.exist?(entry.name)
|
119
107
|
entry.extract(entry.name)
|
120
108
|
end
|
121
109
|
end
|
122
110
|
ensure
|
123
|
-
Dir.chdir(
|
111
|
+
Dir.chdir(saved_dir)
|
124
112
|
end
|
125
113
|
|
126
|
-
|
127
114
|
describe Oddb2xml::RefdataDownloader do
|
128
115
|
include ServerMockHelper
|
129
116
|
before(:all) do
|
130
117
|
VCR.eject_cassette
|
131
118
|
VCR.configure do |c|
|
132
119
|
c.before_record(:Refdata_DE) do |i|
|
133
|
-
if
|
134
|
-
puts "#{Time.now}: #{__LINE__}: Parsing response.body (#{i.response.body.size/(1024*1024)} MB ) will take some time. URI was #{i.request.uri}"
|
120
|
+
if !/WSDL$/.match(i.request.uri) && /refdatabase.refdata.ch\/Service/.match(i.request.uri) && (i.response.body.size > 1024 * 1024)
|
121
|
+
puts "#{Time.now}: #{__LINE__}: Parsing response.body (#{i.response.body.size / (1024 * 1024)} MB ) will take some time. URI was #{i.request.uri}"
|
135
122
|
doc = REXML::Document.new(i.response.body)
|
136
123
|
items = doc.root.children.first.elements.first
|
137
|
-
|
138
|
-
puts "#{Time.now}: #{__LINE__}: Removing most of the #{
|
139
|
-
|
140
|
-
items.elements.each{
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
items.delete x unless x.elements['GTIN'] and Oddb2xml::GTINS_DRUGS.index(x.elements['GTIN'].text)
|
124
|
+
nr_items = doc.root.children.first.elements.first.elements.size
|
125
|
+
puts "#{Time.now}: #{__LINE__}: Removing most of the #{nr_items} items will take some time"
|
126
|
+
nr_searched = 0
|
127
|
+
items.elements.each { |x|
|
128
|
+
nr_searched += 1
|
129
|
+
puts "#{Time.now}: #{__LINE__}: nr_searched #{nr_searched}/#{nr_items}" if nr_searched % 1000 == 0
|
130
|
+
items.delete x unless x.elements["GTIN"] && Oddb2xml::GTINS_DRUGS.index(x.elements["GTIN"].text)
|
145
131
|
}
|
146
132
|
i.response.body = doc.to_s
|
147
|
-
puts "#{Time.now}: response.body is now #{i.response.body.size/(1024*1024)} MB long"
|
148
|
-
i.response.headers[
|
133
|
+
puts "#{Time.now}: response.body is now #{i.response.body.size / (1024 * 1024)} MB long"
|
134
|
+
i.response.headers["Content-Length"] = i.response.body.size
|
149
135
|
end
|
150
136
|
end
|
151
137
|
end
|
152
|
-
VCR.insert_cassette(
|
138
|
+
VCR.insert_cassette("oddb2xml", tag: :Refdata_DE)
|
153
139
|
common_before
|
154
140
|
end
|
155
141
|
after(:all) do
|
156
142
|
common_after
|
157
143
|
end
|
158
|
-
context
|
144
|
+
context "Pharma" do
|
159
145
|
before(:all) do
|
160
146
|
@downloader = Oddb2xml::RefdataDownloader.new({}, :pharma)
|
161
147
|
@xml = @downloader.download
|
162
148
|
end
|
163
|
-
it_behaves_like
|
164
|
-
context
|
165
|
-
it
|
149
|
+
it_behaves_like "any downloader"
|
150
|
+
context "when download_by is called" do
|
151
|
+
it "should parse response hash to xml" do
|
166
152
|
expect(@xml).to be_a String
|
167
153
|
expect(@xml.length).not_to eq(0)
|
168
154
|
expect(@xml).to match(XML_VERSION_1_0)
|
169
155
|
end
|
170
|
-
it
|
156
|
+
it "should return valid xml" do
|
171
157
|
expect(@xml).to match(/PHAR/)
|
172
158
|
expect(@xml).to match(/ITEM/)
|
173
159
|
end
|
174
160
|
end
|
175
161
|
end
|
176
162
|
|
177
|
-
context
|
178
|
-
it_behaves_like
|
163
|
+
context "NonPharma" do
|
164
|
+
it_behaves_like "any downloader"
|
179
165
|
before(:all) do
|
180
166
|
@downloader = Oddb2xml::RefdataDownloader.new({}, :nonpharma)
|
181
167
|
@xml = @downloader.download
|
182
168
|
end
|
183
|
-
context
|
184
|
-
it
|
169
|
+
context "when download_by is " do
|
170
|
+
it "should parse response hash to xml" do
|
185
171
|
expect(@xml).to be_a String
|
186
172
|
expect(@xml.length).not_to eq(0)
|
187
173
|
expect(@xml).to match(XML_VERSION_1_0)
|
188
174
|
end
|
189
|
-
it
|
175
|
+
it "should return valid xml" do
|
190
176
|
expect(@xml).to match(/NONPHAR/)
|
191
177
|
expect(@xml).to match(/ITEM/)
|
192
178
|
end
|
@@ -196,40 +182,45 @@ end
|
|
196
182
|
|
197
183
|
def cleanPackungenXlsx(info)
|
198
184
|
m = /dokumente\/liste/i.match(info.request.uri)
|
199
|
-
puts "#{Time.now}: #{__LINE__} SwissmedicDownloader #{info.request.uri} #{m[1]} (#{info.response.body.size/(1024*1024)} MB )."
|
185
|
+
puts "#{Time.now}: #{__LINE__} SwissmedicDownloader #{info.request.uri} #{m[1]} (#{info.response.body.size / (1024 * 1024)} MB )."
|
200
186
|
return unless m
|
201
187
|
name = nil
|
202
|
-
name =
|
203
|
-
name =
|
204
|
-
swissmedic_dir = File.join(Oddb2xml::
|
188
|
+
name = "packungen" if /zugelasseneverpackungen/.match?(info.request.uri)
|
189
|
+
name = "orphan" if /zugelasseneverpackungen/.match?(info.request.uri)
|
190
|
+
swissmedic_dir = File.join(Oddb2xml::WORK_DIR, "swissmedic")
|
205
191
|
FileUtils.makedirs(swissmedic_dir)
|
206
|
-
xlsx_name = File.join(swissmedic_dir, name +
|
207
|
-
if /Packungen/i.match(xlsx_name)
|
208
|
-
FileUtils.rm(xlsx_name, :
|
209
|
-
File.open(xlsx_name,
|
210
|
-
FileUtils.cp(xlsx_name, File.join(Oddb2xml::SpecData,
|
192
|
+
xlsx_name = File.join(swissmedic_dir, name + ".xlsx")
|
193
|
+
if /Packungen/i.match?(xlsx_name)
|
194
|
+
FileUtils.rm(xlsx_name, verbose: false) if File.exist?(xlsx_name)
|
195
|
+
File.open(xlsx_name, "wb+") { |f| f.write(info.response.body) }
|
196
|
+
FileUtils.cp(xlsx_name, File.join(Oddb2xml::SpecData, "swissmedic_package_downloaded.xlsx"), verbose: true, preserve: true)
|
211
197
|
puts "#{Time.now}: #{__LINE__}: Openening saved #{xlsx_name} (#{File.size(xlsx_name)} bytes) will take some time. URI was #{info.request.uri}"
|
212
198
|
workbook = RubyXL::Parser.parse(xlsx_name)
|
213
199
|
worksheet = workbook[0]
|
214
200
|
drugs = []
|
215
|
-
Oddb2xml::GTINS_DRUGS.each
|
216
|
-
|
201
|
+
Oddb2xml::GTINS_DRUGS.each do |x|
|
202
|
+
next unless x.to_s.size == 13
|
203
|
+
drugs << [x.to_s[4..8].to_i, x.to_s[9..11].to_i]
|
204
|
+
end
|
205
|
+
idx = 6
|
206
|
+
to_delete = []
|
217
207
|
puts "#{Time.now}: Finding items to delete will take some time"
|
218
|
-
while
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
208
|
+
while worksheet.sheet_data[idx]
|
209
|
+
idx += 1
|
210
|
+
next unless worksheet.sheet_data[idx - 1][Oddb2xml::COLUMNS_FEBRUARY_2019.keys.index(:iksnr)]
|
211
|
+
to_delete << (idx - 1) unless drugs.find { |x|
|
212
|
+
(x[0] == worksheet.sheet_data[idx - 1][Oddb2xml::COLUMNS_FEBRUARY_2019.keys.index(:iksnr)].value.to_i) &&
|
213
|
+
(x[1] == worksheet.sheet_data[idx - 1][Oddb2xml::COLUMNS_FEBRUARY_2019.keys.index(:ikscd)].value.to_i)
|
214
|
+
}
|
224
215
|
end
|
225
216
|
if to_delete.size > 0
|
226
217
|
puts "#{Time.now}: Deleting #{to_delete.size} of the #{idx} items will take some time"
|
227
|
-
to_delete.
|
218
|
+
to_delete.reverse_each { |row_id| worksheet.delete_row(row_id) }
|
228
219
|
workbook.write(xlsx_name)
|
229
|
-
FileUtils.cp(xlsx_name, File.join(Oddb2xml::SpecData,
|
220
|
+
FileUtils.cp(xlsx_name, File.join(Oddb2xml::SpecData, "swissmedic_package_shortened.xlsx"), verbose: true, preserve: true)
|
230
221
|
info.response.body = IO.binread(xlsx_name)
|
231
|
-
info.response.headers[
|
232
|
-
puts "#{Time.now}: response.body is now #{info.response.body.size/(1024*1024)} MB long. #{xlsx_name} was #{File.size(xlsx_name)}"
|
222
|
+
info.response.headers["Content-Length"] = info.response.body.size
|
223
|
+
puts "#{Time.now}: response.body is now #{info.response.body.size / (1024 * 1024)} MB long. #{xlsx_name} was #{File.size(xlsx_name)}"
|
233
224
|
end
|
234
225
|
end
|
235
226
|
end
|
@@ -240,79 +231,77 @@ describe Oddb2xml::SwissmedicDownloader do
|
|
240
231
|
mock_downloads
|
241
232
|
VCR.configure do |c|
|
242
233
|
c.before_record(:swissmedic) do |i|
|
243
|
-
|
244
|
-
info = i
|
245
|
-
begin
|
246
|
-
if i.response.headers['Content-Disposition'] and /www.swissmedic.ch/.match(i.request.uri) and i.response.body.size > 1024*1024
|
234
|
+
if i.response.headers["Content-Disposition"] && /www.swissmedic.ch/.match(i.request.uri) && (i.response.body.size > 1024 * 1024)
|
247
235
|
cleanPackungenXlsx(i)
|
248
236
|
end
|
249
|
-
|
250
|
-
|
251
|
-
|
237
|
+
rescue => error
|
238
|
+
puts error if $VERBOSE
|
239
|
+
# require "pry"; binding.pry
|
252
240
|
end
|
253
241
|
end
|
254
242
|
end
|
255
|
-
# 2015-06-10 18:54:40 UTC: SwissmedicDownloader attachment; filename="Zugelassene_Packungen_310515.xlsx" (785630 bytes). URI was https://www.swissmedic.ch/arzneimittel/00156/00221/00222/00230/index.html?download=NHzLpZeg7t,lnp6I0NTU042l2Z6ln1acy4Zn4Z2qZpnO2Yuq2Z6gpJCDdHx7hGym162epYbg2c_JjKbNoKSn6A--&lang=de
|
243
|
+
# 2015-06-10 18:54:40 UTC: SwissmedicDownloader attachment; filename="Zugelassene_Packungen_310515.xlsx" (785630 bytes). URI was https://www.swissmedic.ch/arzneimittel/00156/00221/00222/00230/index.html?download=NHzLpZeg7t,lnp6I0NTU042l2Z6ln1acy4Zn4Z2qZpnO2Yuq2Z6gpJCDdHx7hGym162epYbg2c_JjKbNoKSn6A--&lang=de
|
256
244
|
|
257
|
-
context
|
245
|
+
context "orphan" do
|
258
246
|
before(:each) do
|
259
247
|
VCR.configure do |c|
|
260
248
|
c.before_record(:swissmedic) do |i|
|
261
|
-
|
262
|
-
info = i
|
263
|
-
begin
|
264
|
-
if i.response.headers['Content-Disposition'] and /www.swissmedic.ch/.match(i.request.uri) and i.response.body.size > 1024*1024
|
249
|
+
if i.response.headers["Content-Disposition"] && /www.swissmedic.ch/.match(i.request.uri) && (i.response.body.size > 1024 * 1024)
|
265
250
|
cleanPackungenXlsx(i)
|
266
251
|
end
|
267
|
-
|
268
|
-
|
269
|
-
|
252
|
+
rescue => error
|
253
|
+
puts error if $VERBOSE
|
254
|
+
# require "pry"; binding.pry
|
270
255
|
end
|
271
256
|
end
|
272
257
|
VCR.eject_cassette
|
273
|
-
VCR.insert_cassette(
|
258
|
+
VCR.insert_cassette("oddb2xml", tag: :swissmedic, exclusive: false)
|
259
|
+
FileUtils.rm_rf(Oddb2xml::DOWNLOADS, verbose: true)
|
274
260
|
common_before
|
275
261
|
@downloader = Oddb2xml::SwissmedicDownloader.new(:orphan)
|
276
262
|
end
|
277
|
-
after(:each)
|
278
|
-
it_behaves_like
|
279
|
-
context
|
263
|
+
after(:each) { common_after }
|
264
|
+
it_behaves_like "any downloader"
|
265
|
+
context "download_by for orphan xls" do
|
280
266
|
let(:bin) {
|
281
267
|
@downloader.download
|
282
268
|
}
|
283
|
-
it
|
269
|
+
it "should return valid Binary-String" do
|
284
270
|
# unless [:orphan, :package].index(@downloader.type)
|
285
|
-
|
286
|
-
|
271
|
+
expect(bin).to be_a String
|
272
|
+
expect(bin.bytes).not_to be nil
|
287
273
|
# end
|
288
274
|
end
|
289
|
-
it
|
275
|
+
it "should clean up current directory" do
|
290
276
|
unless [:orphan, :package].index(@downloader.type)
|
291
277
|
expect { bin }.not_to raise_error
|
292
|
-
expect(File.exist?(
|
278
|
+
expect(File.exist?("oddb_orphan.xls")).to eq(false)
|
293
279
|
end
|
280
|
+
expect(File.dirname(bin)).to be == (Oddb2xml::DOWNLOADS)
|
281
|
+
expect(File.exist?(bin)).to eq(true)
|
294
282
|
end
|
295
|
-
it
|
296
|
-
expect(File.exist?(
|
283
|
+
it "should save into the download directory" do
|
284
|
+
expect(File.exist?(bin)).to eq(true)
|
285
|
+
expect(File.exist?(File.join(Oddb2xml::DOWNLOADS, "swissmedic_orphan.xlsx"))).to eq(true)
|
297
286
|
end
|
298
287
|
end
|
299
288
|
end
|
300
|
-
context
|
289
|
+
context "package" do
|
301
290
|
before(:each) do
|
302
291
|
VCR.eject_cassette
|
303
|
-
VCR.insert_cassette(
|
292
|
+
VCR.insert_cassette("oddb2xml", tag: :swissmedic, exclusive: false)
|
304
293
|
common_before
|
305
294
|
@downloader = Oddb2xml::SwissmedicDownloader.new(:package)
|
306
295
|
@bin = @downloader.download
|
307
296
|
end
|
308
|
-
after(:each)
|
309
|
-
context
|
310
|
-
it
|
297
|
+
after(:each) { common_after }
|
298
|
+
context "download_by for package xls" do
|
299
|
+
it "should return valid Binary-String" do
|
311
300
|
expect(@bin).to be_a String
|
312
301
|
expect(@bin.bytes).not_to be nil
|
313
302
|
end
|
314
|
-
it
|
315
|
-
expect(File.exist?(File.join(Oddb2xml::
|
303
|
+
it "should save into the download directory" do
|
304
|
+
expect(File.exist?(File.join(Oddb2xml::DOWNLOADS, "swissmedic_package.xlsx"))).to eq(true)
|
316
305
|
end
|
317
306
|
end
|
318
307
|
end
|
@@ -327,59 +316,59 @@ describe Oddb2xml::EphaDownloader do
|
|
327
316
|
Oddb2xml.add_epha_changes_for_ATC(1, 3, force_run: true)
|
328
317
|
@csv = @downloader.download
|
329
318
|
}
|
330
|
-
expect(File.exist?(File.join(Oddb2xml::
|
319
|
+
expect(File.exist?(File.join(Oddb2xml::DOWNLOADS, "epha_interactions.csv"))).to eq(true)
|
331
320
|
end
|
332
321
|
after(:all) do
|
333
322
|
common_after
|
334
323
|
end
|
335
324
|
|
336
|
-
it_behaves_like
|
325
|
+
it_behaves_like "any downloader"
|
337
326
|
|
338
|
-
context
|
339
|
-
let(:csv) { @csv
|
340
|
-
it
|
327
|
+
context "when download is called" do
|
328
|
+
let(:csv) { @csv }
|
329
|
+
it "should read csv as String" do
|
341
330
|
expect(csv).to be_a String
|
342
331
|
expect(csv.bytes).not_to be nil
|
343
332
|
end
|
344
|
-
it
|
345
|
-
expect(File.exist?(
|
333
|
+
it "should clean up current directory" do
|
334
|
+
expect(File.exist?("epha_interactions.csv")).to eq(false)
|
346
335
|
end
|
347
|
-
it
|
348
|
-
expect(File.exist?(File.join(Oddb2xml::
|
336
|
+
it "should save under download" do
|
337
|
+
expect(File.exist?(File.join(Oddb2xml::DOWNLOADS, "epha_interactions.csv"))).to eq(true)
|
349
338
|
end
|
350
339
|
end
|
351
340
|
end
|
352
341
|
|
353
342
|
describe Oddb2xml::BagXmlDownloader do
|
354
343
|
include ServerMockHelper
|
355
|
-
before(:all)
|
344
|
+
before(:all) { VCR.eject_cassette }
|
356
345
|
before(:all) {
|
357
346
|
VCR.configure do |c|
|
358
347
|
c.before_record(:bag_xml) do |i|
|
359
|
-
if i.response.headers[
|
360
|
-
bag_dir = File.join(Oddb2xml::
|
361
|
-
FileUtils.makedirs(Oddb2xml::
|
362
|
-
tmp_zip = File.join(Oddb2xml::
|
363
|
-
File.open(tmp_zip,
|
348
|
+
if i.response.headers["Content-Disposition"] && /XMLPublications.zip/.match(i.request.uri)
|
349
|
+
bag_dir = File.join(Oddb2xml::WORK_DIR, "bag")
|
350
|
+
FileUtils.makedirs(Oddb2xml::WORK_DIR)
|
351
|
+
tmp_zip = File.join(Oddb2xml::WORK_DIR, "XMLPublications.zip")
|
352
|
+
File.open(tmp_zip, "wb+") { |f| f.write(i.response.body) }
|
364
353
|
unzip_files(tmp_zip, bag_dir)
|
365
354
|
bag_tmp = File.join(bag_dir, PREP_XML)
|
366
355
|
puts "#{Time.now}: #{__LINE__}: Parsing #{File.size(bag_tmp)} (#{File.size(bag_tmp)} bytes) will take some time. URI was #{i.request.uri}"
|
367
356
|
doc = REXML::Document.new(File.read(bag_tmp))
|
368
357
|
items = doc.root.elements
|
369
358
|
puts "#{Time.now}: Removing most of the #{items.size} items will take some time"
|
370
|
-
items.each{ |x| items.delete x unless
|
371
|
-
File.open(bag_tmp,
|
359
|
+
items.each { |x| items.delete x unless Oddb2xml::GTINS_DRUGS.index(x.elements["Packs/Pack/GTIN"].text); }
|
360
|
+
File.open(bag_tmp, "wb+") { |f| f.write(doc.to_s.gsub(/\n\s+\n/, "\n")) }
|
372
361
|
puts "Saved #{bag_tmp} (#{File.size(tmp_zip)} bytes)"
|
373
362
|
zip_files(tmp_zip, Dir.glob("#{bag_dir}/*"))
|
374
363
|
puts "Saved #{tmp_zip} (#{File.size(tmp_zip)} bytes)"
|
375
364
|
i.response.body = IO.binread(tmp_zip)
|
376
|
-
i.response.headers[
|
377
|
-
puts "#{Time.now}: response.body is now #{i.response.body.size/(1024*1024)} MB long. #{tmp_zip} was #{File.size(tmp_zip)}"
|
365
|
+
i.response.headers["Content-Length"] = i.response.body.size
|
366
|
+
puts "#{Time.now}: response.body is now #{i.response.body.size / (1024 * 1024)} MB long. #{tmp_zip} was #{File.size(tmp_zip)}"
|
378
367
|
end
|
379
368
|
end
|
380
369
|
end
|
381
|
-
|
382
|
-
|
370
|
+
VCR.eject_cassette
|
371
|
+
VCR.use_cassette("oddb2xml", tag: :bag_xml) do
|
383
372
|
@downloader = Oddb2xml::BagXmlDownloader.new
|
384
373
|
end
|
385
374
|
common_before
|
@@ -388,19 +377,19 @@ describe Oddb2xml::BagXmlDownloader do
|
|
388
377
|
common_after
|
389
378
|
end
|
390
379
|
|
391
|
-
it_behaves_like
|
392
|
-
context
|
380
|
+
it_behaves_like "any downloader"
|
381
|
+
context "when download is called" do
|
393
382
|
let(:xml) {
|
394
383
|
VCR.eject_cassette
|
395
|
-
VCR.use_cassette(
|
384
|
+
VCR.use_cassette("oddb2xml", tag: :bag_xml) do
|
396
385
|
@downloader.download
|
397
386
|
end
|
398
387
|
}
|
399
|
-
it
|
388
|
+
it "should parse zip to string" do
|
400
389
|
expect(xml).to be_a String
|
401
390
|
expect(xml.length).not_to eq(0)
|
402
391
|
end
|
403
|
-
it
|
392
|
+
it "should return valid xml" do
|
404
393
|
expect(xml).to match(XML_VERSION_1_0)
|
405
394
|
expect(xml).to match(/Preparations/)
|
406
395
|
expect(xml).to match(/DescriptionDe/)
|
@@ -410,174 +399,177 @@ end
|
|
410
399
|
|
411
400
|
describe Oddb2xml::LppvDownloader do
|
412
401
|
include ServerMockHelper
|
413
|
-
before(:all)
|
402
|
+
before(:all) { VCR.eject_cassette }
|
414
403
|
before(:all) do
|
415
|
-
VCR.insert_cassette(
|
404
|
+
VCR.insert_cassette("oddb2xml", tag: :lppv)
|
416
405
|
common_before
|
417
406
|
@downloader = Oddb2xml::LppvDownloader.new
|
418
407
|
@text = @downloader.download
|
419
408
|
end
|
420
|
-
after(:each)
|
409
|
+
after(:each) { common_after }
|
421
410
|
|
422
|
-
it_behaves_like
|
423
|
-
context
|
411
|
+
it_behaves_like "any downloader"
|
412
|
+
context "when download is called" do
|
424
413
|
let(:txt) { @downloader.download }
|
425
|
-
it
|
414
|
+
it "should read txt as String" do
|
426
415
|
expect(@text).to be_a String
|
427
416
|
expect(@text.bytes).not_to be nil
|
428
417
|
end
|
429
418
|
end
|
430
419
|
end
|
431
420
|
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
after(:each) do common_after end
|
442
|
-
|
443
|
-
it_behaves_like 'any downloader'
|
444
|
-
context 'when download is called' do
|
445
|
-
let(:bin) { @downloader.download }
|
446
|
-
it 'should read xls as Binary-String' do
|
447
|
-
expect(bin).to be_a String
|
448
|
-
expect(bin.bytes).not_to be nil
|
421
|
+
unless SKIP_MIGEL_DOWNLOADER
|
422
|
+
describe Oddb2xml::MigelDownloader do
|
423
|
+
include ServerMockHelper
|
424
|
+
before(:all) { VCR.eject_cassette }
|
425
|
+
before(:each) do
|
426
|
+
@downloader = Oddb2xml::MigelDownloader.new
|
427
|
+
VCR.insert_cassette("oddb2xml", tag: :migel)
|
428
|
+
common_before
|
429
|
+
@downloader.download
|
449
430
|
end
|
450
|
-
|
451
|
-
|
452
|
-
|
431
|
+
after(:each) { common_after }
|
432
|
+
|
433
|
+
it_behaves_like "any downloader"
|
434
|
+
context "when download is called" do
|
435
|
+
let(:bin) { @downloader.download }
|
436
|
+
it "should read xls as Binary-String" do
|
437
|
+
expect(bin).to be_a String
|
438
|
+
expect(bin.bytes).not_to be nil
|
439
|
+
end
|
440
|
+
it "should clean up current directory" do
|
441
|
+
expect { bin }.not_to raise_error
|
442
|
+
expect(File.exist?("oddb2xml_files_nonpharma.txt")).to eq(false)
|
443
|
+
end
|
453
444
|
end
|
454
445
|
end
|
455
|
-
end
|
446
|
+
end
|
456
447
|
|
457
448
|
describe Oddb2xml::ZurroseDownloader do
|
458
449
|
include ServerMockHelper
|
459
|
-
before(:all)
|
450
|
+
before(:all) { VCR.eject_cassette }
|
460
451
|
before(:each) do
|
461
452
|
VCR.configure do |c|
|
462
453
|
c.before_record(:zurrose) do |i|
|
463
|
-
if /pillbox/i.match(i.request.uri)
|
454
|
+
if /pillbox/i.match?(i.request.uri)
|
464
455
|
puts "#{Time.now}: #{__LINE__}: URI was #{i.request.uri}"
|
465
|
-
tmp_zip = File.join(Oddb2xml::SpecData,
|
456
|
+
tmp_zip = File.join(Oddb2xml::SpecData, "vcr", "transfer.zip")
|
466
457
|
i.response.body = IO.binread(tmp_zip)
|
467
|
-
i.response.headers[
|
458
|
+
i.response.headers["Content-Length"] = i.response.body.size
|
468
459
|
end
|
469
460
|
end
|
470
461
|
end
|
471
|
-
VCR.insert_cassette(
|
462
|
+
VCR.insert_cassette("oddb2xml", tag: :zurrose)
|
472
463
|
@downloader = Oddb2xml::ZurroseDownloader.new
|
473
464
|
common_before
|
474
465
|
@downloader.download
|
475
466
|
end
|
476
|
-
after(:each)
|
467
|
+
after(:each) { common_after }
|
477
468
|
|
478
|
-
it_behaves_like
|
479
|
-
context
|
469
|
+
it_behaves_like "any downloader"
|
470
|
+
context "when download is called" do
|
480
471
|
let(:dat) { @downloader.download }
|
481
|
-
it
|
472
|
+
it "should read dat as String" do
|
482
473
|
expect(dat).to be_a String
|
483
474
|
expect(dat.bytes).not_to be nil
|
484
475
|
end
|
485
|
-
it
|
476
|
+
it "should clean up current directory" do
|
486
477
|
expect { dat }.not_to raise_error
|
487
|
-
expect(File.exist?(
|
488
|
-
expect(File.exist?(
|
489
|
-
expect(File.exist?(
|
478
|
+
expect(File.exist?("transfer.dat")).to eq(false)
|
479
|
+
expect(File.exist?("oddb2xml_zurrose_transfer.dat")).to eq(false)
|
480
|
+
expect(File.exist?("transfer.zip")).to eq(false)
|
490
481
|
end
|
491
|
-
it
|
492
|
-
expect(File.exist?(File.join(Oddb2xml::
|
493
|
-
expect(File.exist?(File.join(Oddb2xml::
|
482
|
+
it "should save into the download directory" do
|
483
|
+
expect(File.exist?(File.join(Oddb2xml::DOWNLOADS, "transfer.zip"))).to eq(true)
|
484
|
+
expect(File.exist?(File.join(Oddb2xml::DOWNLOADS, "transfer.dat"))).to eq(true)
|
494
485
|
end
|
495
486
|
end
|
496
487
|
end
|
497
488
|
|
498
489
|
describe Oddb2xml::MedregbmDownloader do
|
499
490
|
include ServerMockHelper
|
500
|
-
before(:all)
|
491
|
+
before(:all) { VCR.eject_cassette }
|
501
492
|
before(:each) do
|
502
493
|
VCR.configure do |c|
|
503
494
|
c.before_record(:medreg) do |i|
|
504
|
-
if /medregbm.admin.ch/i.match(i.request.uri)
|
505
|
-
puts "#{Time.now}: #{__LINE__}: URI was #{i.request.uri} containing #{i.response.body.size/(1024*1024)} MB "
|
495
|
+
if /medregbm.admin.ch/i.match?(i.request.uri)
|
496
|
+
puts "#{Time.now}: #{__LINE__}: URI was #{i.request.uri} containing #{i.response.body.size / (1024 * 1024)} MB "
|
506
497
|
begin
|
507
|
-
medreg_dir = File.join(Oddb2xml::
|
498
|
+
medreg_dir = File.join(Oddb2xml::WORK_DIR, "medreg")
|
508
499
|
FileUtils.makedirs(medreg_dir)
|
509
|
-
xlsx_name = File.join(medreg_dir, /ListBetrieb/.match(i.request.uri) ?
|
510
|
-
File.open(xlsx_name,
|
500
|
+
xlsx_name = File.join(medreg_dir, /ListBetrieb/.match?(i.request.uri) ? "Betriebe.xlsx" : "Personen.xlsx")
|
501
|
+
File.open(xlsx_name, "wb+") { |f| f.write(i.response.body) }
|
511
502
|
puts "#{Time.now}: Openening saved #{xlsx_name} (#{File.size(xlsx_name)} bytes) will take some time. URI was #{i.request.uri}"
|
512
503
|
workbook = RubyXL::Parser.parse(xlsx_name)
|
513
504
|
worksheet = workbook[0]
|
514
|
-
idx = 1
|
515
|
-
|
505
|
+
idx = 1
|
506
|
+
to_delete = []
|
507
|
+
while worksheet.sheet_data[idx]
|
516
508
|
idx += 1
|
517
|
-
next unless worksheet.sheet_data[idx-1][0]
|
518
|
-
to_delete << (idx-1) unless Oddb2xml::GTINS_MEDREG.index(worksheet.sheet_data[idx-1][0].value.to_i)
|
509
|
+
next unless worksheet.sheet_data[idx - 1][0]
|
510
|
+
to_delete << (idx - 1) unless Oddb2xml::GTINS_MEDREG.index(worksheet.sheet_data[idx - 1][0].value.to_i)
|
519
511
|
end
|
520
512
|
if to_delete.size > 0
|
521
513
|
puts "#{Time.now}: Deleting #{to_delete.size} of the #{idx} items will take some time"
|
522
|
-
to_delete.
|
514
|
+
to_delete.reverse_each { |row_id| worksheet.delete_row(row_id) }
|
523
515
|
workbook.write(xlsx_name)
|
524
516
|
i.response.body = IO.binread(xlsx_name)
|
525
|
-
i.response.headers[
|
526
|
-
puts "#{Time.now}: response.body is now #{i.response.body.size/(1024*1024)} MB long. #{xlsx_name} was #{File.size(xlsx_name)}"
|
517
|
+
i.response.headers["Content-Length"] = i.response.body.size
|
518
|
+
puts "#{Time.now}: response.body is now #{i.response.body.size / (1024 * 1024)} MB long. #{xlsx_name} was #{File.size(xlsx_name)}"
|
527
519
|
end
|
528
520
|
rescue
|
529
521
|
puts "Creating empty content, as I am unable to parse the XLSX file"
|
530
522
|
i.response.body = ""
|
531
|
-
i.response.headers[
|
523
|
+
i.response.headers["Content-Length"] = i.response.body.size
|
532
524
|
end
|
533
525
|
end
|
534
526
|
end
|
535
527
|
end
|
536
528
|
common_before
|
537
529
|
end
|
538
|
-
after(:each)
|
530
|
+
after(:each) { common_after }
|
539
531
|
|
540
|
-
context
|
532
|
+
context "betrieb" do
|
541
533
|
before(:each) do
|
542
534
|
VCR.eject_cassette
|
543
|
-
VCR.insert_cassette(
|
535
|
+
VCR.insert_cassette("oddb2xml", tag: :medreg)
|
544
536
|
@downloader = Oddb2xml::MedregbmDownloader.new(:company)
|
545
537
|
@downloader.download
|
546
538
|
end
|
547
|
-
after(:each)
|
548
|
-
it_behaves_like
|
549
|
-
context
|
539
|
+
after(:each) { common_after }
|
540
|
+
it_behaves_like "any downloader"
|
541
|
+
context "download betrieb txt" do
|
550
542
|
let(:txt) { @downloader.download }
|
551
|
-
it
|
543
|
+
it "should return valid String" do
|
552
544
|
expect(txt).to be_a String
|
553
545
|
expect(txt.bytes).not_to be nil
|
554
546
|
end
|
555
|
-
it
|
547
|
+
it "should clean up current directory" do
|
556
548
|
expect { txt }.not_to raise_error
|
557
|
-
expect(File.exist?(
|
549
|
+
expect(File.exist?("oddb_company.xls")).to eq(false)
|
558
550
|
end
|
559
551
|
end
|
560
552
|
end
|
561
553
|
|
562
|
-
context
|
554
|
+
context "person" do
|
563
555
|
before(:each) do
|
564
556
|
VCR.eject_cassette
|
565
|
-
VCR.insert_cassette(
|
557
|
+
VCR.insert_cassette("oddb2xml", tag: :medreg)
|
566
558
|
@downloader = Oddb2xml::MedregbmDownloader.new(:person)
|
567
559
|
end
|
568
|
-
after(:each)
|
569
|
-
context
|
560
|
+
after(:each) { common_after }
|
561
|
+
context "download person txt" do
|
570
562
|
let(:txt) {
|
571
563
|
# this downloads a xlsx file (2.5MB), where we should keep only the first few lines
|
572
564
|
@downloader.download
|
573
565
|
}
|
574
|
-
it
|
566
|
+
it "should return valid String" do
|
575
567
|
expect(txt).to be_a String
|
576
568
|
expect(txt.bytes).not_to be nil
|
577
569
|
end
|
578
|
-
it
|
570
|
+
it "should clean up current directory" do
|
579
571
|
expect { txt }.not_to raise_error
|
580
|
-
expect(File.exist?(
|
572
|
+
expect(File.exist?("oddb_person.xls")).to eq(false)
|
581
573
|
end
|
582
574
|
end
|
583
575
|
end
|
@@ -588,49 +580,48 @@ describe Oddb2xml::SwissmedicInfoDownloader do
|
|
588
580
|
before(:all) do
|
589
581
|
VCR.configure do |c|
|
590
582
|
c.before_record(:swissmedicInfo) do |i|
|
591
|
-
|
592
|
-
|
593
|
-
|
594
|
-
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
602
|
-
|
583
|
+
puts "#{Time.now}: #{__LINE__}: URI was #{i.request.uri} returning #{i.response.body.size / (1024 * 1024)} MB"
|
584
|
+
if i.response.headers["Content-Disposition"]
|
585
|
+
m = /filename=([^\d]+)/.match(i.response.headers["Content-Disposition"][0])
|
586
|
+
if m
|
587
|
+
name = m[1].chomp("_")
|
588
|
+
if /AipsDownload/i.match?(name)
|
589
|
+
# we replace this by manually reduced xml file from spec/data
|
590
|
+
# As we only use to create the fachinfo, we don't need many elements
|
591
|
+
tmp_zip = File.join(Oddb2xml::SpecData, "AipsDownload.zip")
|
592
|
+
i.response.body = IO.binread(tmp_zip)
|
593
|
+
i.response.headers["Content-Length"] = i.response.body.size
|
594
|
+
puts "#{Time.now}: #{__LINE__}: response.body is now #{i.response.body.size / (1024 * 1024)} MB long. #{tmp_zip} was #{File.size(tmp_zip)}"
|
595
|
+
end
|
603
596
|
end
|
604
597
|
end
|
605
598
|
end
|
606
599
|
end
|
607
|
-
end
|
608
600
|
VCR.eject_cassette
|
609
|
-
VCR.insert_cassette(
|
601
|
+
VCR.insert_cassette("oddb2xml", tag: :swissmedicInfo)
|
610
602
|
common_before
|
611
603
|
@downloader = Oddb2xml::SwissmedicInfoDownloader.new
|
612
604
|
@downloader.download
|
613
605
|
end
|
614
|
-
after(:all)
|
615
|
-
it_behaves_like
|
616
|
-
context
|
617
|
-
let(:xml) { @downloader.download
|
618
|
-
it
|
606
|
+
after(:all) { common_after }
|
607
|
+
it_behaves_like "any downloader"
|
608
|
+
context "when download is called" do
|
609
|
+
let(:xml) { @downloader.download }
|
610
|
+
it "should parse zip to String" do
|
619
611
|
expect(xml).to be_a String
|
620
612
|
expect(xml.length).not_to eq(0)
|
621
613
|
end
|
622
|
-
it
|
614
|
+
it "should return valid xml" do
|
623
615
|
expect(xml).to match(XML_VERSION_1_0)
|
624
616
|
expect(xml).to match(/medicalInformations/)
|
625
617
|
expect(xml).to match(/content/)
|
626
618
|
end
|
627
|
-
it
|
619
|
+
it "should clean up current directory" do
|
628
620
|
expect { xml }.not_to raise_error
|
629
|
-
expect(File.exist?(
|
621
|
+
expect(File.exist?("swissmedic_info.zip")).to eq(false)
|
630
622
|
end
|
631
|
-
it
|
632
|
-
expect(File.exist?(File.join(Oddb2xml::
|
623
|
+
it "should save into the download directory" do
|
624
|
+
expect(File.exist?(File.join(Oddb2xml::DOWNLOADS, "swissmedic_info.zip"))).to eq(true)
|
633
625
|
end
|
634
626
|
end
|
635
627
|
end
|
636
|
-
|