oddb2xml 2.5.0 → 2.5.1

data/lib/oddb2xml/cli.rb

@@ -11,7 +11,7 @@ require 'rubyXL'
 require 'date' # for today
 
 module Oddb2xml
-  
+
   class Cli
     attr_reader :options
     SUBJECTS  = %w[product article]
@@ -19,6 +19,7 @@ module Oddb2xml
     OPTIONALS = %w[fi fi_product]
     def initialize(args)
       @options = args
+      STDOUT.puts "\nStarting cli with from #{caller[1]} using #{@options}" if defined?(RSpec)
       Oddb2xml.save_options(@options)
       @mutex = Mutex.new
       # product
@@ -38,6 +39,7 @@ module Oddb2xml
     end
     def run
       threads = []
+      startTime = Time.now
       files2rm = Dir.glob(File.join(Downloads, '*'))
       FileUtils.rm_f(files2rm, :verbose => @options[:log]) if files2rm.size > 0 and not Oddb2xml.skip_download?
       if @options[:calc] and not @options[:extended]
@@ -79,46 +81,56 @@ module Oddb2xml
         exit
       end
       build
+      if @options[:artikelstamm]
+        elexis_v5_xsd = File.expand_path(File.join(__FILE__, '..', '..', '..', 'Elexis_Artikelstamm_v5.xsd'))
+         cmd = "xmllint --noout --schema #{elexis_v5_xsd} #{@_files[:artikelstamm]}"
+         if system(cmd)
+            puts "Validatied #{@_files[:artikelstamm]}"
+         else
+            puts "Validating failed using #{cmd}"
+            exit(2)
+         end
+      end
       compress if @options[:compress_ext]
-      report
+      res = report
+      nrSecs =  (Time.now - startTime).to_i
+      if defined?(RSpec) && (nrSecs).to_i > 10 then require 'pry'; binding.pry ; end
+      res
     end
     private
     def build
-      Oddb2xml.log("Start build")
       begin
-        @_files = {"calc"=>"oddb_calc.xml"} if @options[:calc] and not @options[:extended]
-        files.each_pair do |sbj, file|
-          builder = Builder.new(@options) do |builder|
-            if @options[:calc] and not @options[:extended]
-              builder.packs = @packs
-              builder.subject = sbj
-            elsif @options[:address]
-              builder.subject   = sbj
-              builder.companies = @companies
-              builder.people    = @people
-            else # product
-              if @options[:format] != :dat
-                refdata = {}
-                types.each do |type|
-                  refdata.merge!(@refdata_types[type]) if @refdata_types[type]
-                end
-                builder.refdata = refdata
-                builder.subject = sbj
-              end
-              # common sources
-              builder.items = @items
-              builder.flags = @flags
-              builder.lppvs = @lppvs
-              # optional sources
-              builder.infos = @infos
-              builder.packs = @packs
-              # additional sources
-              %w[actions orphan migel infos_zur_rose].each do |addition|
-                builder.send("#{addition}=".intern, self.instance_variable_get("@#{addition}"))
+        @_files = {"calc"=>"oddb_calc.xml"} if @options[:calc] and not (@options[:extended] || @options[:artikelstamm])
+        builder = Builder.new(@options) do |builder|
+          if @options[:calc] and not  (@options[:extended] || @options[:artikelstamm])
+            builder.packs = @packs
+          elsif @options[:address]
+            builder.companies = @companies
+            builder.people    = @people
+          else # product
+            if @options[:format] != :dat
+              refdata = {}
+              types.each do |type|
+                refdata.merge!(@refdata_types[type]) if @refdata_types[type]
               end
+              builder.refdata = refdata
+            end
+            # common sources
+            builder.items = @items
+            builder.flags = @flags
+            builder.lppvs = @lppvs
+            # optional sources
+            builder.infos = @infos
+            builder.packs = @packs
+            # additional sources
+            %w[actions orphan migel infos_zur_rose].each do |addition|
+              builder.send("#{addition}=".intern, self.instance_variable_get("@#{addition}"))
             end
-            builder.tag_suffix = @options[:tag_suffix]
           end
+          builder.tag_suffix = @options[:tag_suffix]
+        end
+        files.each_pair do |sbj, file|
+          builder.subject = sbj
           output = ''
           if !@options[:address] and (@options[:format] == :dat)
             types.each do |type|
@@ -187,14 +199,14 @@ module Oddb2xml
       when :orphan
         var = what.to_s
         begin # instead of Thread.new do
-          downloader = SwissmedicDownloader.new(what)
+          downloader = SwissmedicDownloader.new(what, @options)
           bin = downloader.download
           Oddb2xml.log("SwissmedicDownloader #{var} #{bin} #{File.size(bin)} bytes")
           self.instance_variable_set(
             "@#{var}",
             items = SwissmedicExtractor.new(bin, what).to_arry
           )
-          Oddb2xml.log("SwissmedicExtractor added #{items.size} #{var}. File #{bin} was #{File.size(bin)} bytes")
+          Oddb2xml.log("SwissmedicExtractor added #{items.size}")
           items
         end
       when :interaction
@@ -259,7 +271,7 @@ module Oddb2xml
           xml = downloader.download
           Oddb2xml.log("ZurroseDownloader xml #{xml.size} bytes")
           @mutex.synchronize do
-            hsh = ZurroseExtractor.new(xml, @options[:extended]).to_hash
+            hsh = ZurroseExtractor.new(xml, @options[:extended], @options[:artikelstamm]).to_hash
             Oddb2xml.log("ZurroseExtractor added #{hsh.size} items from xml with #{xml.size} bytes")
             @infos_zur_rose = hsh
           end
@@ -302,7 +314,9 @@ module Oddb2xml
       unless @_files
         @_files = {}
         @_files[:calc] = "oddb_calc.xml" if @options[:calc]
-        if @options[:address]
+        if @options[:artikelstamm]
+          @_files[:artikelstamm]           = "artikelstamm_#{Date.today.strftime('%d%m%Y')}_v5.xml"
+        elsif @options[:address]
           @_files[:company] = "#{prefix}_betrieb.xml"
           @_files[:person]  = "#{prefix}_medizinalperson.xml"
         elsif @options[:format] == :dat
@@ -336,41 +350,47 @@ module Oddb2xml
         lines << Calc.report_conversion
         lines << ParseComposition.report
       end
-      unless @options[:address]
-        types.each do |type|
-          if @refdata_types[type]
-            indices = @refdata_types[type].values.flatten.length
-            if type == :nonpharma
-              nonpharmas = @refdata_types[type].keys
-              if SkipMigelDownloader
-                indices + nonpharmas.length
+      if  @options[:artikelstamm]
+        lines << "Generated artikelstamm.xml for Elexis"
+        lines += Builder.articlestamm_v5_info_lines
+      else
+        unless @options[:address]
+          types.each do |type|
+            if @refdata_types[type]
+              indices = @refdata_types[type].values.flatten.length
+
+              if type == :nonpharma
+                nonpharmas = @refdata_types[type].keys
+                if SkipMigelDownloader
+                  indices + nonpharmas.length
+                else
+                  migel_xls  = @migel.values.compact.select{|m| !m[:pharmacode]}.map{|m| m[:pharmacode] }
+                  indices += (migel_xls - nonpharmas).length # ignore duplicates, null
+                end
+                lines << sprintf("\tNonPharma products: %i", indices)
               else
-                migel_xls  = @migel.values.compact.select{|m| !m[:pharmacode]}.map{|m| m[:pharmacode] }
-                indices += (migel_xls - nonpharmas).length # ignore duplicates, null
+                lines << sprintf("\tPharma products: %i", indices)
               end
-              lines << sprintf("\tNonPharma products: %i", indices)
-            else
-              lines << sprintf("\tPharma products: %i", indices)
             end
           end
-        end
-        if @options[:extended]
-          lines << sprintf("\tInformation items zur Rose: %i", @infos_zur_rose.length)
-        end
-      else
-        {
-          'Betrieb' => :@companies,
-          'Person'  => :@people
-        }.each do |type, var|
-          lines << sprintf(
-            "#{type} addresses: %i", self.instance_variable_get(var).length)
+          if  (@options[:extended] || @options[:artikelstamm])
+            lines << sprintf("\tInformation items zur Rose: %i", @infos_zur_rose.length)
+          end
+        else
+          {
+            'Betrieb' => :@companies,
+            'Person'  => :@people
+          }.each do |type, var|
+            lines << sprintf(
+              "#{type} addresses: %i", self.instance_variable_get(var).length)
+          end
         end
       end
       puts lines.join("\n")
     end
     def types # RefData
       @_types ||=
-        if @options[:nonpharma]
+        if @options[:nonpharma] || @options[:artikelstamm]
           [:pharma, :nonpharma]
         else
           [:pharma]

data/lib/oddb2xml/compare.rb

@@ -0,0 +1,189 @@
+# encoding: utf-8
+require 'xmlsimple'
+
+module Oddb2xml
+  def self.log_timestamp(msg)
+    full_msg = "#{Time.now.strftime("%H:%M:%S")}: #{msg}"
+    puts full_msg
+    STDOUT.flush
+    full_msg
+  end
+  class StammXML
+    V3_NAME_REG = /_([N,P])_/
+    attr_accessor :components
+    attr_reader :keys, :sub_key_names, :filename, :basename, :version, :hash
+    def initialize(filename, components =  ['ITEMS'])
+      raise "File #{filename} must exist" unless File.exist?(filename)
+      @filename = filename
+      @basename = File.basename(filename)
+      @version =  V3_NAME_REG.match(filename) ? 3 : 5
+      @components = components
+      if @version == 5
+        @hash = load_file(@filename)
+      else
+        raise "Unsupported version #{@version}"
+      end
+    end
+    def self.get_component_key_name(component_name)
+        return 'LIMNAMEBAG' if /LIMITATION/i.match(component_name)
+        return 'PRODNO' if /PRODUCT/i.match(component_name)
+        return 'GTIN' if /ITEM/i.match(component_name)
+        raise "Cannot determine keyname for component #{component_name}"
+    end
+    def get_limitation_from_v5(item)
+      get_item('PRODUCTS', item['PRODNO'].first.to_i)['LIMNAMEBAG'] ? ['true'] : nil
+    end
+    def get_field_from_v5_product(item, field_name)
+      get_item('PRODUCTS', item['PRODNO'].first.to_i)[field_name]
+    end
+    def get_items(component_name)
+      if @version == 3
+        items = @hash[component_name]
+      else
+        items = @hash[component_name].first.values.first
+      end
+      items
+    end
+    def get_item(component_name, id)
+      keyname = StammXML.get_component_key_name(component_name)
+      get_items(component_name).find{|item| item[keyname].first.to_i == id}
+    end
+    def load_file(name)
+      Oddb2xml.log_timestamp "Reading #{name} #{(File.size(name)/1024/1024).to_i} MB. This may take some time"
+      XmlSimple.xml_in(IO.read(name))
+    end
+  end
+  class CompareV5
+    DEFAULTS = {
+      :components => ["PRODUCTS", "LIMITATIONS", "ITEMS",],
+      :fields_to_ignore => ['COMP', 'DOSAGE_FORMF', 'MEASUREF'],
+      :fields_as_floats => [ 'PEXT', 'PEXF', 'PPUB' ],
+      :min_diff_for_floats => 0.01,
+    }
+    def initialize(left, right, options = DEFAULTS.clone)
+      @options = options
+      @left = StammXML.new(left, @options[:components])
+      @right = StammXML.new(right, @options[:components])
+      @diff_stat = {}
+      @occurrences = {}
+      @report = []
+    end
+    def get_keys(items, key='GTIN')
+      items.collect{|item| item[key].first.to_i }
+    end
+    def get_names(items)
+       items.collect{|item| item.keys}.flatten.uniq.sort
+    end
+    def compare
+      show_header("Start comparing #{@left.filename} with #{@right.filename}")
+      (@left.components & @right.components).each do |name|
+        begin
+          puts "\n#{Time.now.strftime("%H:%M:%S")}: Comparing #{name} in #{@left.basename} with #{@right.basename}"
+          key = StammXML.get_component_key_name(name)
+          left_items = @left.get_items(name)
+          next unless left_items
+          right_items = @right.get_items(name)
+          next unless right_items
+          @diff_stat[name] = {}
+          @occurrences[name] = {}
+          @diff_stat[name][NR_COMPARED] = 0
+          l_names = get_names(left_items)
+          r_names = get_names(right_items)
+          compare_names = l_names & r_names
+          l_keys = get_keys(left_items, key)
+          r_keys = get_keys(right_items, key)
+          (l_keys & r_keys).each do |id|
+            compare_details(name, compare_names, id)
+          end
+          key_results_details(name, compare_names, l_keys, r_keys)
+        rescue => error
+          puts "Execution failed with #{error}"
+        end
+      end
+      show_header("Summary comparing #{@left.filename} with #{@right.filename}")
+      puts "Ignored differences in #{@options[:fields_to_ignore]}. Signaled when differences in #{@options[:fields_as_floats]} were bigger than #{@options[:min_diff_for_floats]}"
+      puts @report.join("\n")
+      @diff_stat.each do |component, stats|
+        puts "\nFor #{stats[NR_COMPARED]} #{component} we have the following number of differences per field"
+        stats.each do |name, nr|
+          next if name.eql?(NR_COMPARED)
+          next if @options[:fields_to_ignore].index(name)
+          puts "   #{name.ljust(20)} #{nr} of #{@occurrences[component][name]}"
+        end
+      end
+      @diff_stat
+    rescue => error
+      puts "Execution failed with #{error}"
+      raise error
+    end
+    private
+    NR_COMPARED = 'NR_COMPARED'
+    COUNT       = '_count'
+    def show_header(header)
+      text = Oddb2xml.log_timestamp(header)
+      pad = 5
+      puts
+      puts '-'*(text.length+2*pad)
+      puts ''.ljust(pad) + text
+      puts '-'*(text.length+2*pad)
+      puts
+    end
+    def compare_details(component_name, compare_names, id)
+      l_item = @left.get_item(component_name, id)
+      r_item = @right.get_item(component_name, id)
+      found_one = false
+      length = 32
+      found = false
+      detail_name = l_item['DSCR'] ? l_item['DSCR'].first[0..length-1].rjust(length) : ''.rjust(length)
+      details = "Diff in #{id.to_s.ljust(15)} #{detail_name}"
+      diff_name = component_name
+      diff_name += 'S' unless /S$/.match(diff_name)
+      @diff_stat[diff_name] ||= {}
+      @occurrences[diff_name] ||= {}
+      @diff_stat[diff_name][NR_COMPARED] ||= 0
+      @diff_stat[diff_name][NR_COMPARED] += 1
+      l_item.keys.each do |sub_key|
+        next if @options[:fields_to_ignore].index(sub_key)
+        @diff_stat[diff_name][sub_key] ||= 0
+        @occurrences[diff_name][sub_key] ||= 0
+        @occurrences[diff_name][sub_key] += 1
+        r_value = r_item[sub_key]
+        l_value = l_item[sub_key]
+        if @options[:fields_as_floats].index(sub_key)
+          l_float = l_value ? l_value.first.to_f : 0.0
+          r_float = r_value ? r_value.first.to_f : 0.0
+          next if (l_float - r_float).abs < @options[:min_diff_for_floats]
+        end
+        next if (r_value.is_a?(Array) && '--missing--'.eql?(r_value.first)) || (l_value.is_a?(Array) && '--missing--'.eql?(l_value.first))
+                # TODO: get_field_from_v5_product
+        next if r_value.to_s.eql?(l_value.to_s)
+        next if r_value.to_s.upcase.eql?(l_value.to_s.upcase) && @options[:case_insensitive]
+        details += " #{sub_key}: '#{l_value}' != '#{r_value}'"
+        found = found_one = true
+        @diff_stat[diff_name][sub_key] += 1
+      end
+      puts details.gsub(/[\[\]]/,'') if found
+    end
+
+    def show_keys(keys, batch_size = 20)
+      0.upto(keys.size) do |idx|
+        next unless idx % batch_size == 0
+        puts '  ' + keys[idx..(idx + batch_size-1)].join(' ')
+      end
+    end
+    def key_results_details(component_name, compare_names, l_keys, r_keys)
+      component_name += 'S' unless /S$/.match(component_name)
+      @report <<  "#{component_name}: Found #{l_keys.size} items only in #{@left.basename} #{r_keys.size} items only in #{@right.basename}, compared #{@diff_stat[component_name][NR_COMPARED]} items"
+      keys = r_keys - l_keys
+      head = "#{component_name}: #{(keys).size} keys only in #{@right.basename}"
+      puts "#{head}: Keys were #{keys.size}"
+      show_keys(keys)
+      @report << head
+      keys = l_keys - r_keys
+      head = "#{component_name}: #{(keys).size} keys only in #{@left.basename}"
+      puts "#{head}: Keys were #{keys.size}"
+      show_keys(keys)
+      @report << head
+    end
+  end
+end

data/lib/oddb2xml/compressor.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 require 'zlib'
-require 'archive/tar/minitar'
+require 'minitar'
 require 'zip'
 
 module Oddb2xml
@@ -35,9 +35,12 @@ module Oddb2xml
             end
           end
         end
-        if File.exists? @compress_file and not defined?(Rspec)
+        if File.exists? @compress_file
+          puts "#{__LINE__}: @compress_file"
           @contents.each do |file|
-            FileUtils.rm(file)
+            @tmpfile = file
+          puts "#{__LINE__}: @tmpfile"
+            FileUtils.rm(file) if file && File.exists?(file)
           end
         end
         rescue Errno::ENOENT, StandardError => e

data/lib/oddb2xml/downloader.rb

@@ -12,24 +12,21 @@ SkipMigelDownloader = true  # https://github.com/zdavatz/oddb2xml_files/raw/mast
 module Oddb2xml
   module DownloadMethod
     private
-    def download_as(file, option='r')
+    def download_as(file, option='w+')
       tempFile  = File.join(WorkDir,   File.basename(file))
-      file2save = File.join(Downloads, File.basename(file))
-      Oddb2xml.log "download_as file #{file2save} via #{tempFile} from #{@url}"
+      @file2save = File.join(Downloads, File.basename(file))
+      report_download(@url, @file2save)
       data = nil
-      FileUtils.rm_f(tempFile, :verbose => false)
       if Oddb2xml.skip_download(file)
         io = File.open(file, option)
         data = io.read
       else
         begin
-          response = @agent.get(@url)
-          response.save_as(file)
-          response = nil # win
           io = File.open(file, option)
-          data = io.read
-        rescue Timeout::Error, Errno::ETIMEDOUT
-          retrievable? ? retry : raise
+          data = open(@url).read
+          io.write(data)
+        rescue => error
+          puts "error #{error} while fetching #{@url}"
         ensure
           io.close if io and !io.closed? # win
           Oddb2xml.download_finished(tempFile)
@@ -39,7 +36,7 @@ module Oddb2xml
     end
   end
   class Downloader
-    attr_reader :type, :agent
+    attr_reader :type, :agent, :url; :file2save
     def initialize(options={}, url=nil)
       @options     = options
       @url         = url
@@ -48,6 +45,12 @@ module Oddb2xml
       Oddb2xml.log "Downloader from #{@url} for #{self.class}"
       init
     end
+    def report_download(url, file)
+      Oddb2xml.log sprintf("%-20s: download_as %-24s from %s",
+                           self.class.to_s.split('::').last,
+                           File.basename(file),
+                           url)
+    end
     def init
       @agent = Mechanize.new
       @agent.user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:16.0) Gecko/20100101 Firefox/16.0'
@@ -79,6 +82,7 @@ module Oddb2xml
         Dir.glob(File.join(Downloads, '*')).each { |name| if target.match(name) then entry = name; break end }
         if entry
           dest = "#{Downloads}/#{File.basename(entry)}"
+          @file2save = dest
           if File.exists?(dest)
             Oddb2xml.log "read_xml_from_zip return content of #{dest} #{File.size(dest)} bytes "
             return IO.read(dest)
@@ -133,8 +137,9 @@ module Oddb2xml
     include DownloadMethod
     def download
       @url ||= 'https://download.epha.ch/cleaned/matrix.csv'
-      content = download_as('epha_interactions.csv', 'r')
-      FileUtils.rm_f('epha_interactions.csv', :verbose => true)
+      file = 'epha_interactions.csv'
+      content = download_as(file, 'w+')
+      FileUtils.rm_f(file, :verbose => false)
       content
     end
   end
@@ -142,36 +147,27 @@ module Oddb2xml
     include DownloadMethod
     def download
       @url ||= 'https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/LPPV.txt'
-      download_as('oddb2xml_files_lppv.txt', 'r')
+      download_as('oddb2xml_files_lppv.txt', 'w+')
     end
   end
   class ZurroseDownloader < Downloader
     include DownloadMethod
     def download
       @url ||= 'http://pillbox.oddb.org/TRANSFER.ZIP'
-      unless @url =~ /^http/
-        io = File.open(@url, 'r:iso-8859-1:utf-8')
-        content = io.read
-        Oddb2xml.log("ZurroseDownloader #{__LINE__} download #{@url} @url returns #{content.bytes}")
-        content
-      else
-        file = File.join(Downloads, 'transfer.zip')
-        unless Oddb2xml.skip_download(file)
-          Oddb2xml.log "ZurroseDownloader #{__LINE__}: #{file}"
-          begin
-            response = @agent.get(@url)
-            response.save_as(file)
-            response = nil # win
-          rescue Timeout::Error, Errno::ETIMEDOUT
-            retrievable? ? retry : raise
-          ensure
-            Oddb2xml.download_finished(file)
-          end
-        end
-        read_xml_from_zip(/transfer.dat/, file)
-        dest = File.join(Downloads, 'transfer.dat')
-        File.open(dest, 'r:iso-8859-1:utf-8').read
+      zipfile = File.join(WorkDir, 'transfer.zip')
+      download_as(zipfile)
+      dest = File.join(Downloads, 'transfer.dat')
+      cmd = "unzip -o '#{zipfile}' -d '#{Downloads}'"
+      system(cmd)
+      if @options[:artikelstamm]
+        cmd = "iconv -f ISO8859-1 -t utf-8 -o #{dest.sub('.dat','.utf8')} #{dest}"
+        Oddb2xml.log(cmd)
+        system(cmd)
       end
+      # read file and convert it to utf-8
+      File.open(dest, 'r:iso-8859-1:utf-8').read
+    ensure
+      FileUtils.rm(zipfile) if File.exist?(dest) && File.exist?(zipfile)
     end
   end
   class MedregbmDownloader < Downloader
@@ -190,30 +186,35 @@ module Oddb2xml
       super({}, url)
     end
     def download
-      download_as("medregbm_#{@type.to_s}.txt", 'r:iso-8859-1:utf-8')
+      file = "medregbm_#{@type.to_s}.txt"
+      download_as(file, 'w+:iso-8859-1:utf-8')
+      report_download(@url, file)
+      FileUtils.rm_f(file, :verbose => false) # we need it only in the download
+      file
     end
   end
   class BagXmlDownloader < Downloader
+    include DownloadMethod
     def init
       super
       @url ||= 'http://bag.e-mediat.net/SL2007.Web.External/File.axd?file=XMLPublications.zip'
     end
     def download
       file = File.join(WorkDir, 'XMLPublications.zip')
-      Oddb2xml.log "BagXmlDownloader #{__LINE__}: #{file}"
-      unless Oddb2xml.skip_download(file)
-        Oddb2xml.log "BagXmlDownloader #{__LINE__}: #{file}"
-        begin
-          response = @agent.get(@url)
-          response.save_as(file)
-          response = nil # win
-        rescue Timeout::Error, Errno::ETIMEDOUT
-          retrievable? ? retry : raise
-        ensure
-          Oddb2xml.download_finished(file)
-        end
+      download_as(file)
+      report_download(@url, file)
+      if defined?(RSpec)
+        src = File.join(Oddb2xml::SpecData, 'Preparations.xml')
+        content =  File.read(src)
+        FileUtils.cp(src, File.join(Downloads, File.basename(file)))
+      else
+        content = read_xml_from_zip(/Preparations.xml/, File.join(Downloads, File.basename(file)))
+      end
+      if @options[:artikelstamm]
+        cmd = "xmllint --format --output Preparations.xml Preparations.xml"
+        Oddb2xml.log(cmd)
+        system(cmd)
       end
-      content = read_xml_from_zip(/Preparations.xml/, File.join(Downloads, File.basename(file)))
       FileUtils.rm_f(file, :verbose => false) unless defined?(RSpec)
       content
     end
@@ -236,7 +237,7 @@ module Oddb2xml
     def download
       begin
         filename =  "refdata_#{@type}.xml"
-        file2save = File.join(Downloads, "refdata_#{@type}.xml")
+        @file2save = File.join(Downloads, "refdata_#{@type}.xml")
 				soap = %(<?xml version="1.0" encoding="UTF-8"?>
 <SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns1="http://refdatabase.refdata.ch/Article_in" xmlns:ns2="http://refdatabase.refdata.ch/">
   <SOAP-ENV:Body>
@@ -247,14 +248,21 @@ module Oddb2xml
   </SOAP-ENV:Envelope>
 </ns1:ATYPE></ns2:DownloadArticleInput></SOAP-ENV:Body>
 )
-        return IO.read(file2save) if Oddb2xml.skip_download? and File.exists?(file2save)
-        FileUtils.rm_f(file2save, :verbose => false)
+        report_download(@url, @file2save)
+        return IO.read(@file2save) if Oddb2xml.skip_download? and File.exists?(@file2save)
+        FileUtils.rm_f(@file2save, :verbose => false)
         response = @client.call(:download, :xml => soap)
         if response.success?
           if xml = response.to_xml
+            xml =  File.read(File.join(Oddb2xml::SpecData, File.basename(@file2save))) if defined?(RSpec)
             response = nil # win
             FileUtils.makedirs(Downloads)
-            File.open(file2save, 'w+') { |file| file.write xml }
+            File.open(@file2save, 'w+') { |file| file.write xml }
+            if @options[:artikelstamm]
+              cmd = "xmllint --format --output #{@file2save} #{@file2save}"
+              Oddb2xml.log(cmd)
+              system(cmd)
+            end
           else
             # received broken data or internal error
             raise StandardError
@@ -271,6 +279,7 @@ module Oddb2xml
     end
   end
   class SwissmedicDownloader < Downloader
+    include DownloadMethod
     def initialize(type=:orphan, options = {})
       @type = type
       @options = options
@@ -282,23 +291,28 @@ module Oddb2xml
       end
     end
     def download
-      @type == file = File.join(Oddb2xml::WorkDir, "swissmedic_#{@type}.xlsx")
-      if  @options[:calc] and @options[:skip_download] and File.exists?(file) and (Time.now-File.ctime(file)).to_i < 24*60*60
-        Oddb2xml.log "SwissmedicDownloader #{__LINE__}: Skip downloading #{file} #{File.size(file)} bytes"
-        return File.expand_path(file)
+      @file2save = File.join(Oddb2xml::WorkDir, "swissmedic_#{@type}.xlsx")
+      report_download(@url, @file2save)
+      if  @options[:calc] and @options[:skip_download] and File.exists?(@file2save) and (Time.now-File.ctime(@file2save)).to_i < 24*60*60
+        Oddb2xml.log "SwissmedicDownloader #{__LINE__}: Skip downloading #{@file2save} #{File.size(@file2save)} bytes"
+          return File.expand_path(@file2save)
       end
       begin
-        FileUtils.rm(File.expand_path(file), :verbose => !defined?(RSpec)) if File.exists?(File.expand_path(file))
-        File.open(file, 'w+') do |output|
-          output.write open(@direct_url_link).read
+        FileUtils.rm(File.expand_path(@file2save), :verbose => !defined?(RSpec)) if File.exists?(File.expand_path(@file2save))
+        @url = @direct_url_link
+        download_as(@file2save, 'w+')
+        if @options[:artikelstamm]
+          cmd = "ssconvert '#{@file2save}' '#{File.join(Downloads, File.basename(@file2save).sub(/\.xls.*/, '.csv'))}' 2> /dev/null"
+          Oddb2xml.log(cmd)
+          system(cmd)
         end
-        return File.expand_path(file)
+        return File.expand_path(@file2save)
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       ensure
-        Oddb2xml.download_finished(file, false)
+        Oddb2xml.download_finished(@file2save, false)
       end
-      return File.expand_path(file)
+      return File.expand_path(@file2save)
     end
   end
   class SwissmedicInfoDownloader < Downloader
@@ -309,6 +323,7 @@ module Oddb2xml
     end
     def download
       file = File.join(Downloads, "swissmedic_info.zip")
+      report_download(@url, file)
       FileUtils.rm_f(file, :verbose => false) unless Oddb2xml.skip_download?
       begin
         response = nil