oddb2xml 1.8.1 → 1.8.2

data/.travis.yml

@@ -6,7 +6,7 @@ before_install:
   - gem update --system 2.1.11
   - gem --version
 
-script: bundle exec rake spec
+script: bundle exec rake test
 
 rvm:
   - 1.9.3

data/Gemfile.lock

@@ -1,15 +1,15 @@
 PATH
   remote: .
   specs:
-    oddb2xml (1.8.1)
-      archive-tar-minitar
+    oddb2xml (1.8.2)
+      archive-tar-minitar (~> 0.5.2)
       mechanize (~> 2.5.1)
       nokogiri (~> 1.5.10)
       rubyXL (~> 2.5)
-      rubyzip (~> 1.0)
-      savon (~> 2.0)
-      sax-machine
-      spreadsheet
+      rubyzip (~> 1.1.3)
+      savon (~> 2.4.0)
+      sax-machine (~> 0.1.0)
+      spreadsheet (~> 0.9.7)
 
 GEM
   remote: https://rubygems.org/
@@ -79,7 +79,7 @@ GEM
       rubyzip (>= 1.0.0)
     rubyntlm (0.3.4)
     rubyzip (1.1.3)
-    safe_yaml (1.0.2)
+    safe_yaml (1.0.3)
     savon (2.4.0)
       akami (~> 1.2.0)
       builder (>= 2.1.2)

data/README.md

@@ -20,7 +20,7 @@ and
 * oddb2xml -f dat
 * oddb2xml -f dat -a nonpharma
 
-creates .dat files according to ([IGM-11](http://dev.ywesee.com/uploads/att/IGM.pdf))
+creates .dat files according to ([IGM-11](http://dev.ywesee.com/uploads/att/IGM.pdf)). IGM-11 describes the structure of the zurrose_transfer.dat.
 
 * oddb.dat
 * oddb_with_migel.dat
@@ -43,7 +43,7 @@ The following additional data is in the files:
 see `--help`.
 
 ```
-/usr/local/bin/oddb2xml ver.1.7.8
+/usr/local/bin/oddb2xml ver.1.8.2
 Usage:
   oddb2xml [option]
     produced files are found under data
@@ -87,6 +87,18 @@ FR
         Pharma products: 14801
 ```
 
+## Supported ruby version
+
+We run tests on travis-ci.org for the Ruby versions mentioned in the .travis.yml file. These are
+* 1.9.3
+* 2.0.0
+* 2.1
+* ruby-head
+
+If you are running an older Ruby-Version (eg. 1.8 or 1.9.1), please upgrade before reporting a bug.
+See also http://bugs.ruby-lang.org/projects/ruby/wiki/ReleaseEngineering
+
+
 ## XSD files
 If you need the XSD files, generate them yourself using the javabeans tool:
 

data/Rakefile

@@ -19,16 +19,25 @@ task :spec => :clean
 
 desc 'Run oddb2xml with all commonly used combinations'
 task :test => [:clean, :spec, :gem] do
-  system("./test_options.rb 2>&1 | tee test_options.log")
+  log_file = 'test_options.log'
+  puts "Running test_options.rb with Output redirected to #{log_file}. This will take some time (e.g. 20 minutes)"
+  # must use bash -o pipefail to catch error in test_options.rb and not tee
+  # see http://stackoverflow.com/questions/985876/tee-and-exit-status
+  res = system("bash -c 'set -o pipefail && ./test_options.rb 2>&1 | tee #{log_file}'")
+  puts "Running test_options.rb returned #{res.inspect}. Output was redirected to #{log_file}"
+  exit 1 unless res
 end
 
 require 'rake/clean'
+CLEAN.include FileList['pkg/*.gem']
+CLEAN.include FileList['*.zip*']
 CLEAN.include FileList['*.xls*']
 CLEAN.include FileList['*.xml*']
 CLEAN.include FileList['*.dat*']
 CLEAN.include FileList['*.tar.gz']
 CLEAN.include FileList['*.txt.*']
 CLEAN.include FileList['*.csv.*']
+CLEAN.include FileList['*.zip.*']
 CLEAN.include FileList['ruby*.tmp']
 CLEAN.include FileList['data/download']
 CLEAN.include FileList['duplicate_ean13_from_zur_rose.txt']

data/bin/oddb2xml

@@ -28,8 +28,8 @@ Usage:
     -x N, --context=N    context N {product|address}. product is default.
                          
                          For debugging purposes
-    --skip-download      skips downloading files it the file is already under data/download.
-                         Downloaded files are saved under data/download
+    --skip-download      skips downloading files it the file is already under downloads.
+                         Downloaded files are saved under downloads
     --log                log important actions
     -h,   --help         Show this help message.
 EOS

data/lib/oddb2xml/builder.rb

@@ -39,6 +39,7 @@ module Oddb2xml
       @companies  = []
       @people     = []
       @tag_suffix = nil
+      @pharmacode = {} # index pharmacode => item
       if block_given?
         yield self
       end
@@ -78,6 +79,7 @@ module Oddb2xml
             obj[:seq] = seq
           end
           @articles << obj
+          @pharmacode[phar] = obj
         end
         # add
         @migel.values.compact.each do |migel|
@@ -117,37 +119,44 @@ module Oddb2xml
           Oddb2xml.log("prepare_articles extended")
           @prices.each{
             |ean13, info|
+          pharmacode = info[:pharmacode]
+          if @pharmacode[pharmacode]
+            @pharmacode[pharmacode][:price]     = info[:price]
+            @pharmacode[pharmacode][:pub_price] = info[:pub_price]
+            next
+          end
           obj = {}
           found = false
           %w[de fr].each do |lang|
           #              existing = @articles.find{|art| art[lang.intern] and art[lang.intern][0][:ean] == ean13 }
-          existing = local_index[lang][ean13]
+
+            existing = local_index[lang][ean13]
             if existing
               found = true
-              existing[:price] = info[:price]
-                                      existing[:pub_price] = info[:pub_price]
-                      else
-                      entry = {
-                               :desc            => info[:description],
-                               :status          => 'I', # or it will not be emitted in the dat
-                              :atc_code        => '',
-                               :ean             => ean13,
-                               :lang            => lang.capitalize,
-                               :pharmacode      => info[:pharmacode],
-                               :price           => info[:price],
-                               :pub_price       => info[:pub_price],
-                               :type            => info[:type],
-                               }
-                      obj[lang.intern] = [entry]
-                      @index[lang.upcase][ean13] = entry
-                      end
-                      end
-                      unless found
-                      @articles << obj
-                      nrAdded += 1
-                      end
-                      }
+              existing[:price]     = info[:price]
+              existing[:pub_price] = info[:pub_price]
+            else
+              entry = {
+                        :desc            => info[:description],
+                        :status          => 'A', # for ZurRose 'A' means aktive, aka available in trade
+                        :atc_code        => '',
+                        :ean             => ean13,
+                        :lang            => lang.capitalize,
+                       :pharmacode      => pharmacode,
+                        :price           => info[:price],
+                        :pub_price       => info[:pub_price],
+                        :type            => info[:type],
+                        }
+              obj[lang.intern] = [entry]
+              @index[lang.upcase][ean13] = entry
+            end
           end
+          unless found
+            @articles << obj
+            nrAdded += 1
+          end
+          }
+        end
       end
       Oddb2xml.log("prepare_articles done. Added #{nrAdded} prices. Total #{@articles.size}")
     end
@@ -604,7 +613,7 @@ module Oddb2xml
           'PROD_DATE'         => datetime,
           'VALID_DATE'        => datetime
         ) {
-          @articles.each do |obj|
+        @articles.each do |obj|
             idx += 1
         Oddb2xml.log "build_article #{idx} of #{@articles.size} articles" if idx % 500 == 0
         obj[:de].each_with_index do |de_idx, i|
@@ -612,6 +621,7 @@ module Oddb2xml
               pac,no8 = nil,de_idx[:ean][4..11] # BAG-XML(SL/LS)
               ppac = nil                        # Packungen
               ean = de_idx[:ean]
+              pharma_code = de_idx[:pharmacode]
               ean = nil if ean.match(/^000000/)
               if obj[:seq]
                 pac = obj[:seq][:packages][de_idx[:pharmacode]]
@@ -777,7 +787,7 @@ module Oddb2xml
                   }
                 end
               }
-            end
+            end if obj[:de]
           end
           xml.RESULT {
             xml.OK_ERROR   'OK'

data/lib/oddb2xml/cli.rb

@@ -9,6 +9,7 @@ require 'oddb2xml/util'
 require 'rubyXL'
 
 module Oddb2xml
+  
   class Cli
     SUBJECTS  = %w[product article]
     ADDITIONS = %w[substance limitation interaction code]
@@ -89,6 +90,7 @@ module Oddb2xml
     end
     private
     def build
+      Oddb2xml.log("Start build")
       begin
         files.each_pair do |sbj, file|
           builder = Builder.new(@options) do |builder|
@@ -141,7 +143,7 @@ module Oddb2xml
           else
             output = builder.to_xml
           end
-          File.open(file, 'w:utf-8'){ |fh| fh << output }
+          File.open(File.join(WorkDir, file), 'w:utf-8'){ |fh| fh << output }
         end
       rescue Interrupt
         files.values.each do |file|
@@ -171,6 +173,7 @@ module Oddb2xml
           @mutex.synchronize do
             hsh = SwissmedicInfoExtractor.new(xml).to_hash
             @infos = hsh
+            Oddb2xml.log("SwissmedicInfoExtractor added #{@infos.size} fachinfo")
           end
         end
       when :orphan, :fridge
@@ -182,6 +185,7 @@ module Oddb2xml
             "@#{var}".intern,
             SwissmedicExtractor.new(bin, what).to_arry
           )
+#          Oddb2xml.log("SwissmedicExtractor added #{self.instance_variable_get("@#{var}".intern).size} #{var}. File #{bin} was #{File.size(bin)} bytes")
         end
       when :interaction
         Thread.new do
@@ -189,6 +193,7 @@ module Oddb2xml
           str = downloader.download
           @mutex.synchronize do
             @actions = EphaExtractor.new(str).to_arry
+            Oddb2xml.log("EphaExtractor added #{@actions.size} interactions")
           end
         end
       when :migel
@@ -197,6 +202,7 @@ module Oddb2xml
           bin = downloader.download
           @mutex.synchronize do
             @migel = MigelExtractor.new(bin).to_hash
+            Oddb2xml.log("MigelExtractor added #{@migel.size} migel items")
           end
         end
       when :package
@@ -205,6 +211,7 @@ module Oddb2xml
           bin = downloader.download
           @mutex.synchronize do
             @packs = SwissmedicExtractor.new(bin, :package).to_hash
+#            Oddb2xml.log("SwissmedicExtractor added #{@packs.size} packs. File #{bin} was #{File.size(bin)} bytes")
           end
         end
       when :bm_update
@@ -213,6 +220,7 @@ module Oddb2xml
           str = downloader.download
           @mutex.synchronize do
             @flags = BMUpdateExtractor.new(str).to_hash
+            Oddb2xml.log("BMUpdateExtractor added #{@flags.size} flags")
           end
         end
       when :lppv
@@ -221,6 +229,7 @@ module Oddb2xml
           str = downloader.download
           @mutex.synchronize do
             @lppvs = LppvExtractor.new(str).to_hash
+            Oddb2xml.log("LppvExtractor added #{@lppvs.size} lppvs")
           end
         end
       when :bag
@@ -230,16 +239,18 @@ module Oddb2xml
           @mutex.synchronize do
             hsh = BagXmlExtractor.new(xml).to_hash
             @items = hsh
+            Oddb2xml.log("BagXmlDownloader added #{@items.size} items")
           end
         end
       when :zurrose
         Thread.new do
           downloader = ZurroseDownloader.new(@options, @options[:transfer_dat])
           xml = downloader.download
+          Oddb2xml.log("zurrose xml #{xml.size} bytes")
           @mutex.synchronize do
             hsh = ZurroseExtractor.new(xml, @options[:extended]).to_hash
             @prices = hsh
-            Oddb2xml.log("zurrose added #{@prices.size} prices")
+            Oddb2xml.log("zurrose added #{@prices.size} prices from xml with #{xml.size} bytes")
           end
         end
       when :index
@@ -265,8 +276,9 @@ module Oddb2xml
     def compress
       compressor = Compressor.new(prefix, @options)
       files.values.each do |file|
-        if File.exists?(file)
-          compressor.contents << file
+        work_file = File.join(WorkDir, file)
+        if File.exists?(work_file)
+          compressor.contents << work_file
         end
       end
       compressor.finalize!
@@ -319,6 +331,9 @@ module Oddb2xml
             end
           end
         end
+        if @options[:extended]
+          lines << sprintf("\tPrices zur Rose: %i", @prices.length)
+        end
       else
         {
           'Betrieb' => :@companies,

data/lib/oddb2xml/compressor.rb

@@ -12,13 +12,14 @@ module Oddb2xml
       @options = options
       @options[:compress_ext] ||= 'tar.gz'
       @options[:format]       ||= :xml
-      @compress_file = "#{prefix}_#{@options[:format].to_s}_" + 
-        Time.now.strftime("%d.%m.%Y_%H.%M.#{@options[:compress_ext]}")
+      @compress_file = "#{prefix}_#{@options[:format].to_s}_" + Time.now.strftime("%d.%m.%Y_%H.%M.#{@options[:compress_ext]}")                                 
+                                 #      @compress_file = File.join(WorkDir, "#{prefix}_#{@options[:format].to_s}_" +
+                                 #Time.now.strftime("%d.%m.%Y_%H.%M.#{@options[:compress_ext]}"))
       @contents = []
       super()
     end
     def finalize!
-      if @contents.empty?
+      if @contents.empty? and @contents.size == 0
         return false
       end
       begin
@@ -34,13 +35,12 @@ module Oddb2xml
             end
           end
         end
-        if File.exists? @compress_file
+        if File.exists? @compress_file and not defined?(Rspec)
           @contents.each do |file|
-            Oddb2xml.download_finished(file)
+            FileUtils.rm(file)
           end
         end
-      rescue Errno::ENOENT, StandardError
-        Oddb2xml.download_finished(@compress_file)
+        rescue Errno::ENOENT, StandardError => e
         return false
       end
       return true

data/lib/oddb2xml/downloader.rb

@@ -8,8 +8,11 @@ module Oddb2xml
   module DownloadMethod
     private
     def download_as(file, option='r')
-      Oddb2xml.log "download_as file #{file} from #{@url}"
+      tempFile  = File.join(WorkDir,   File.basename(file))
+      file2save = File.join(Downloads, File.basename(file))
+      Oddb2xml.log "download_as file #{file2save} via #{tempFile} from #{@url}"
       data = nil
+      FileUtils.rm_f(tempFile, :verbose => false)
       if Oddb2xml.skip_download(file)
         io = File.open(file, option)
         data = io.read
@@ -24,7 +27,7 @@ module Oddb2xml
           retrievable? ? retry : raise
         ensure
           io.close if io and !io.closed? # win
-          Oddb2xml.download_finished(file)
+          Oddb2xml.download_finished(tempFile)
         end
       end
       return data
@@ -63,12 +66,29 @@ module Oddb2xml
         false
       end
     end
-    def read_xml_form_zip(target, zipfile)
+    def read_xml_from_zip(target, zipfile)
+      Oddb2xml.log "read_xml_from_zip target is #{target} zip: #{zipfile} #{File.exists?(zipfile)}"
+      if Oddb2xml.skip_download?
+        entry = nil
+        Dir.glob(File.join(Downloads, '*')).each { |name| if target.match(name) then entry = name; break end }
+        if entry
+          dest = "#{Downloads}/#{File.basename(entry)}"
+          if File.exists?(dest)
+            Oddb2xml.log "read_xml_from_zip return content of #{dest} #{File.size(dest)} bytes "
+            return IO.read(dest)
+          else
+            Oddb2xml.log "read_xml_from_zip could not read #{dest}"
+          end
+        else
+          Oddb2xml.log "read_xml_from_zip could not find #{target.to_s}"
+        end
+      end
       xml = ''
       if RUBY_PLATFORM =~ /mswin|mingw|bccwin|cygwin/i
         Zip::File.open(zipfile) do |zipFile|
           zipFile.each do |entry|
             if entry.name =~ target
+              Oddb2xml.log "read_xml_from_zip reading #{__LINE__}: #{entry.name}"
               io = entry.get_input_stream
               until io.eof?
                 bytes = io.read(1024)
@@ -76,13 +96,20 @@ module Oddb2xml
                 bytes = nil
               end
               io.close if io.respond_to?(:close)
+              dest = "#{Downloads}/#{File.basename(target)}"
+              File.open(dest, 'w+') { |f| f.write xml }
+              Oddb2xml.log "read_xml_from_zip saved as #{dest}"
             end
           end
         end
       else
         Zip::File.foreach(zipfile) do |entry|
           if entry.name =~ target
+            Oddb2xml.log "read_xml_from_zip #{__LINE__}: reading #{entry.name}"
+            dest = "#{Downloads}/#{File.basename(entry.name)}"
             entry.get_input_stream { |io| xml = io.read }
+            File.open(dest, 'w+') { |f| f.write xml }
+            Oddb2xml.log "read_xml_from_zip saved as #{dest}"
           end
         end
       end
@@ -123,9 +150,12 @@ module Oddb2xml
       @url ||= 'http://zurrose.com/fileadmin/main/lib/download.php?file=/fileadmin/user_upload/downloads/ProduktUpdate/IGM11_mit_MwSt/Vollstamm/transfer.dat'
       unless @url =~ /^http/
         io = File.open(@url, 'r:iso-8859-1:utf-8')
-        io.read
+        content = io.read
+        Oddb2xml.log("ZurroseDownloader #{__LINE__} download #{@url} @url returns #{content.bytes}")
+        content   
       else
-        download_as('oddb2xml_zurrose_transfer.dat', 'r:iso-8859-1:utf-8')
+        Oddb2xml.log("ZurroseDownloader #{__LINE__} download #{@url} @url")
+        download_as('zurrose_transfer.dat', 'r:iso-8859-1:utf-8')
       end
     end
   end
@@ -154,30 +184,23 @@ module Oddb2xml
       @url ||= 'http://bag.e-mediat.net/SL2007.Web.External/File.axd?file=XMLPublications.zip'
     end
     def download
-      file = 'XMLPublications.zip'
-      begin
-        if @options[:debug]
-          FileUtils.copy(File.expand_path("../../../spec/data/#{file}", __FILE__), '.')
-        else
-          unless Oddb2xml.skip_download(file)
-            response = @agent.get(@url)
-            response.save_as(file)
-            response = nil # win
-          end
-        end
-        inhalt = read_xml_form_zip(/^Preparation/iu, file)
-        if @options[:skip_download]
-          FileUtils.makedirs(Backup)
-          outfile = File.join(Backup, 'Preparations.xml')
-          Oddb2xml.log "Downloader saving outfile #{outfile} for #{self.class}"
-          File.open(outfile, 'w+') { |file| file.write inhalt}
+      file = File.join(WorkDir, 'XMLPublications.zip')
+      Oddb2xml.log "BagXmlDownloader #{__LINE__}: #{file}"
+      unless Oddb2xml.skip_download(file)
+        Oddb2xml.log "BagXmlDownloader #{__LINE__}: #{file}"                                       
+        begin
+          response = @agent.get(@url)
+          response.save_as(file)
+          response = nil # win
+        rescue Timeout::Error, Errno::ETIMEDOUT
+          retrievable? ? retry : raise
+        ensure
+          Oddb2xml.download_finished(file)
         end
-        inhalt
-      rescue Timeout::Error, Errno::ETIMEDOUT
-        retrievable? ? retry : raise
-      ensure
-        Oddb2xml.download_finished(file)
       end
+      content = read_xml_from_zip(/Preparations.xml/, File.join(Downloads, File.basename(file)))
+      FileUtils.rm_f(file, :verbose => false) unless defined?(Rspec)
+      content
     end
   end
   class SwissIndexDownloader < Downloader
@@ -199,9 +222,10 @@ module Oddb2xml
     end
     def download
       begin
-        if @options[:debug]
-          return File.read(File.expand_path("../../../spec/data/swissindex_#{@type}_#{@lang}.xml", __FILE__))
-        end
+        filename =  "swissindex_#{@type}_#{@lang}.xml"
+        file2save = File.join(Downloads, "swissindex_#{@type}_#{@lang}.xml")
+        return IO.read(file2save) if Oddb2xml.skip_download? and File.exists?(file2save)
+        FileUtils.rm_f(file2save, :verbose => false)
         soap = <<XML
 <?xml version="1.0" encoding="utf-8"?>
 <soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
@@ -214,12 +238,8 @@ XML
         if response.success?
           if xml = response.to_xml
             response = nil # win
-            if @options[:skip_download]
-              file2save = File.expand_path("../../../data/download/swissindex_#{@type}_#{@lang}.xml", __FILE__)
-              FileUtils.makedirs(File.dirname(file2save)) unless File.directory?(File.dirname(file2save))
-              File.open(file2save, 'w+') { |file| file.puts xml }
-            end
-            return xml
+            FileUtils.makedirs(Downloads)
+            File.open(file2save, 'w+') { |file| file.write xml }
           else
             # received broken data or internal error
             raise StandardError
@@ -232,6 +252,7 @@ XML
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       end
+      xml
     end
   end
   class SwissmedicDownloader < Downloader
@@ -257,11 +278,9 @@ XML
         page = @agent.get(@url)
         if link_node = page.search(@xpath).first
           link = Mechanize::Page::Link.new(link_node, @agent, page)
-          unless Oddb2xml.skip_download(file)
-            response = link.click
-            response.save_as(file)
-            response = nil # win
-          end
+          response = link.click
+          response.save_as(file)
+          response = nil # win
         end
         return File.expand_path(file)
       rescue Timeout::Error, Errno::ETIMEDOUT
@@ -269,6 +288,7 @@ XML
       ensure
         Oddb2xml.download_finished(file, false)
       end
+      return File.expand_path(file)
     end
   end
   class SwissmedicInfoDownloader < Downloader
@@ -278,32 +298,31 @@ XML
       @url ||= "http://download.swissmedicinfo.ch/Accept.aspx?ReturnUrl=%2f"
     end
     def download
-      file = "swissmedic_info.zip"
+      file = File.join(Downloads, "swissmedic_info.zip")
+      FileUtils.rm_f(file, :verbose => false) unless Oddb2xml.skip_download?
       begin
-        unless Oddb2xml.skip_download(file)
-          response = nil
-          if home = @agent.get(@url)
-            form = home.form_with(:id => 'Form1')
-            bttn = form.button_with(:name => 'ctl00$MainContent$btnOK')
-            if page = form.submit(bttn)
-              form = page.form_with(:id => 'Form1')
-              bttn = form.button_with(:name => 'ctl00$MainContent$BtnYes')
-              response = form.submit(bttn)
-            end
-          end
-          if response          
-            response.save_as(file)
-            response = nil # win
+        response = nil
+        if home = @agent.get(@url)
+          form = home.form_with(:id => 'Form1')
+          bttn = form.button_with(:name => 'ctl00$MainContent$btnOK')
+          if page = form.submit(bttn)
+            form = page.form_with(:id => 'Form1')
+            bttn = form.button_with(:name => 'ctl00$MainContent$BtnYes')
+            response = form.submit(bttn)
           end
         end
-        read_xml_form_zip(/^AipsDownload_/iu, file)
+        if response
+          response.save_as(file)
+          response = nil # win
+        end
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       rescue NoMethodError
         # pass
       ensure
         Oddb2xml.download_finished(file)
-      end
+      end unless File.exists?(file)
+      read_xml_from_zip(/^AipsDownload_/iu, file)
     end
   end
 end