gtin2atc 0.1.0

data/Rakefile

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'gtin2atc/version'
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+# dependencies are now declared in gtin2atc.gemspec
+
+desc 'Offer a gem task like hoe'
+task :gem => :build do
+  Rake::Task[:build].invoke
+end
+
+task :spec => :clean
+
+desc 'Run gtin2atc with all commonly used combinations'
+task :test => [:clean, :spec, :gem] do
+  log_file = 'test_options.log'
+  puts "Running test_options.rb with Output redirected to #{log_file}. This will take some time (e.g. 20 minutes)"
+  # must use bash -o pipefail to catch error in test_options.rb and not tee
+  # see http://stackoverflow.com/questions/985876/tee-and-exit-status
+  res = system("bash -c 'set -o pipefail && ./test_options.rb 2>&1 | tee #{log_file}'")
+  puts "Running test_options.rb returned #{res.inspect}. Output was redirected to #{log_file}"
+  exit 1 unless res
+end
+
+require 'rake/clean'
+CLEAN.include FileList['pkg/*.gem']
+CLEAN.include FileList['*.zip*']
+CLEAN.include FileList['*.xls*']
+CLEAN.include FileList['*.xml*']
+CLEAN.include FileList['*.dat*']
+CLEAN.include FileList['*.tar.gz']
+CLEAN.include FileList['*.txt.*']
+CLEAN.include FileList['*.csv.*']
+CLEAN.include FileList['*.zip.*']
+CLEAN.include FileList['ruby*.tmp']
+CLEAN.include FileList['data/download']
+CLEAN.include FileList['duplicate_ean13_from_zur_rose.txt']

data/bin/gtin2atc

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+
+root = Pathname.new(__FILE__).realpath.parent.parent
+$:.unshift root.join('lib') if $0 == __FILE__
+
+require 'optparse'
+require 'gtin2atc'
+require 'gtin2atc/builder'
+require "gtin2atc/util"
+
+options = Gtin2atc::Options.new
+
+begin
+  options.parser.parse!(ARGV)
+rescue OptionParser::MissingArgument,
+       OptionParser::InvalidArgument,
+       OptionParser::InvalidOption
+  puts Gtin2atc::Options.help
+  exit
+end
+
+opts = options.opts
+startTime = Time.now
+if ARGV.size == 1 and File.exists?(ARGV[0])
+  args = []
+  IO.readlines(ARGV[0]).each{ |x| args << x.chomp}
+else
+  args = ARGV.clone
+end
+Gtin2atc::Builder.new(opts).run(args)
+diff = (Time.now-startTime).to_i
+Gtin2atc::Util.debug_msg "#{File.basename(__FILE__)} done. Took #{diff} seconds"

data/gtin2atc.gemspec

@@ -0,0 +1,38 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'gtin2atc/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = "gtin2atc"
+  spec.version     = Gtin2atc::VERSION
+  spec.author      = "Niklaus Giger, Zeno R.R. Davatz"
+  spec.email       = "ngiger@ywesee.com, zdavatz@ywesee.com"
+  spec.description = "gtin2atc file with gtin, atc_code, pharmanr from input file with gtin"
+  spec.summary     = "gtin2atc creates csv files with GTIN and ATC."
+  spec.homepage    = "https://github.com/zdavatz/gtin2atc"
+  spec.license       = "GPL-v2"
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  # We fix the version of the spec to newer versions only in the third position
+  # hoping that these version fix only security/severe bugs
+  # Consulted the Gemfile.lock to get 
+  spec.add_dependency 'rubyzip', '~> 1.1.3'
+#  spec.add_dependency 'archive-tar-minitar', '~> 0.5.2'
+  spec.add_dependency 'mechanize', '~> 2.5.1'
+  spec.add_dependency 'nokogiri', '~> 1.5.10'
+  spec.add_dependency 'savon'#, '~> 2.4.0'
+#  spec.add_dependency 'spreadsheet', '~> 1.0.0'
+  spec.add_dependency 'rubyXL'
+  spec.add_dependency 'sax-machine' #,  '~> 0.1.0'
+  
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "rdoc"
+end
+

data/lib/gtin2atc/builder.rb

@@ -0,0 +1,376 @@
+require 'csv'
+require 'rubyXL'
+require "gtin2atc/options"
+require "gtin2atc/downloader"
+require "gtin2atc/xml_definitions"
+require 'mechanize'
+
+module Gtin2atc
+  class Builder
+    Strip_For_Sax_Machine = '<?xml version="1.0" encoding="utf-8"?>'+"\n"
+    def initialize(opts)
+      Util.set_logging(opts[:log])
+      @do_compare = opts[:compare]
+      Util.debug_msg "Builder: opts are #{opts} @do_compare is #{@do_compare}"
+      @data_swissmedic = {}
+      @data_bag = {}
+      @data_swissindex = {}
+      @bag_entries_without_gtin = 0
+    end
+    def calc_checksum(str)
+      str = str.strip
+      sum = 0
+      val =   str.split(//u)
+      12.times do |idx|
+        fct = ((idx%2)*2)+1
+        sum += fct*val[idx].to_i
+      end
+      ((10-(sum%10))%10).to_s
+    end
+    def swissmedic_xls_extractor
+      @swissmedic = SwissmedicDownloader.new
+      filename = @swissmedic.download
+      Util.debug_msg "swissmedic_xls_extractor xml is #{filename}"
+      data = {}
+      @sheet = RubyXL::Parser.parse(File.expand_path(filename)).worksheets[0]
+      i_5,i_3   = 0,10 # :swissmedic_numbers
+      atc       = 5    # :atc_code
+      @sheet.each_with_index do |row, i|
+        next if (i <= 1)
+        next unless row[i_5] and row[i_3]
+        no8 = sprintf('%05d',row[i_5].value.to_i) + sprintf('%03d',row[i_3].value.to_i)
+        unless no8.empty?
+          next if no8.to_i == 0
+          item = {}
+          ean_base12 = "7680#{no8}"
+          gtin = (ean_base12.ljust(12, '0') + calc_checksum(ean_base12)).to_i
+          item = {}
+          item[:gtin]            = gtin
+          item[:atc_code]         =  row[atc] ? row[atc].value.to_s : ''
+          item[:name]             =  row[2].value.to_s
+          data[gtin] = item
+        end
+      end
+      Util.debug_msg "swissmedic_xls_extractor extracted #{data.size} items"
+      data
+    end
+    def swissindex_xml_extractor
+      @swissindex = SwissIndexDownloader.new
+      xml = @swissindex.download
+      Util.debug_msg "swissindex_xml_extractor xml is #{xml.size} bytes long"
+      data = {}
+      result = PharmaEntry.parse(xml.sub(Strip_For_Sax_Machine, ''), :lazy => true)
+      items = result.PHARMA.ITEM
+      items.each do |pac|
+        item = {}
+        gtin = pac.GTIN ? pac.GTIN.to_i : nil
+        next unless item[:gtin].to_i
+        item[:gtin]            = gtin
+        item[:pharmacode]      = (phar = pac.PHAR)   ? phar: ''
+        item[:atc_code]        = (code = pac.ATC)    ? code.to_s : ''
+        item[:description]     = pac.DSCR
+        data[gtin] = item
+      end
+      Util.debug_msg "swissindex_xml_extractor extracted #{data.size} items"
+      data
+    end
+    def bag_xml_extractor
+      data = {}
+      @bag = BagXmlDownloader.new
+      xml = @bag.download
+      Util.debug_msg "bag_xml_extractor xml is #{xml.size} bytes long"
+
+      result = PreparationsEntry.parse(xml.sub(Strip_For_Sax_Machine, ''), :lazy => true)
+      @bag_entries_without_gtin = 0
+      result.Preparations.Preparation.each do |seq|
+        item = {}
+        item[:atc_code]     = (atcc = seq.AtcCode)       ? atcc : ''
+        seq.Packs.Pack.each do |pac|
+          gtin = pac.GTIN
+          if gtin
+            gtin = gtin.to_i
+            item[:gtin] = gtin
+            item[:name] = seq.NameDe + " " +  pac.DescriptionDe
+            data[gtin] = item
+            Util.debug_msg "run_bag_extractor add #{item}" if $VERBOSE
+          else
+            @bag_entries_without_gtin += 1
+            Util.debug_msg "run_bag_extractor skip phar #{seq.NameDe}: #{seq.DescriptionDe} without gtin."
+          end
+        end
+      end
+      Util.debug_msg "bag_xml_extractor extracted #{data.size} items. Skipped #{@bag_entries_without_gtin} entries without gtin"
+      data
+    end
+    def run(gtins_to_parse=[])
+      Util.debug_msg("run #{gtins_to_parse}")
+      Util.debug_msg("@use_swissindex true")
+      @data_swissindex = swissindex_xml_extractor
+      output_name =  File.join(Util.get_archive, @do_compare ? 'gtin2atc_swissindex.csv' : 'gtin2atc.csv')
+      CSV.open(output_name,'w+') do |csvfile|
+        csvfile << ["gtin", "ATC", 'pharmacode', 'description']
+        @data_swissindex.sort.each do |gtin, item|
+          if @do_compare or gtins_to_parse.size == 0 or
+              gtins_to_parse.index(gtin.to_s) or
+              gtins_to_parse.index(item[:pharmacode])
+            csvfile << [gtin, item[:atc_code], item[:pharmacode], item[:description]]
+          end
+        end
+      end
+      msg = "SwissIndex: Extracted #{gtins_to_parse.size} of #{@data_swissindex.size} items into #{output_name} for #{gtins_to_parse}"
+      Util.debug_msg(msg)
+      return unless @do_compare
+      @data_bag = bag_xml_extractor
+      output_name =  File.join(Util.get_archive, 'gtin2atc_bag.csv')
+      CSV.open(output_name,'w+') do |csvfile|
+        csvfile << ["gtin", "ATC", 'description']
+        @data_bag.sort.each do |gtin, item|
+          csvfile << [gtin, item[:atc_code], item[:description]]
+        end
+      end
+      Util.debug_msg "BAG: Extracted #{gtins_to_parse.size} of #{@data_bag.size} items into #{output_name} for #{gtins_to_parse}"
+      @data_swissmedic = swissmedic_xls_extractor
+      output_name =  File.join(Util.get_archive, 'gtin2atc_swissmedic.csv')
+      CSV.open(output_name,'w+') do |csvfile|
+        csvfile << ["gtin", "ATC", 'description']
+        @data_swissmedic.sort.each do |gtin, item|
+          csvfile << [gtin, item[:atc_code], item[:pharmacode], item[:description]]
+        end
+      end
+      Util.debug_msg "SwissMedic: Extracted #{@data_swissmedic.size} items into #{output_name}"
+      check_bag
+      check_swissmedic
+      compare
+    end
+    # require 'pry';
+    def check_bag
+      matching_atc_codes = 0
+
+      not_in_swissmedic = 0
+      match_in_swissmedic = 0
+      shorter_in_swissmedic = 0
+      longer_in_swissmedic = 0
+      different_atc_in_swissmedic = 0
+
+      not_in_swissindex = 0
+      match_in_swissindex = 0
+      shorter_in_swissindex = 0
+      longer_in_swissindex = 0
+      different_atc_in_swissindex = 0
+      j = 0
+      @data_bag.each{
+        |gtin, item|
+        atc_code = item[:atc_code]
+        j += 1
+        Util.debug_msg "#{gtin}: j #{j} checking #{atc_code} in #{item}"
+        if @data_swissmedic[gtin] and @data_swissindex[gtin] and
+          atc_code == @data_swissmedic[gtin][:atc_code] and
+          atc_code == @data_swissindex[gtin][:atc_code]
+          Util.debug_msg "#{gtin}: matching_atc_codes SwissIndex #{item} #{@data_swissmedic[gtin][:atc_code]} and #{@data_swissindex[gtin][:atc_code]}"
+          matching_atc_codes += 1
+          next
+        end
+
+        if not @data_swissindex[gtin]
+          Util.debug_msg "#{gtin}: Not in SwissIndex #{item}"
+          not_in_swissindex += 1
+        elsif atc_code == @data_swissindex[gtin][:atc_code]
+          Util.debug_msg "SwissIndex #{gtin}: ATC code #{atc_code} matches swissindex  #{@data_swissindex[gtin][:atc_code]}"
+          match_in_swissindex += 1
+        elsif atc_code.length < @data_swissindex[gtin][:atc_code].length
+          longer_in_swissindex += 1
+          Util.debug_msg "SwissIndex #{gtin}: ATC code #{atc_code} longer  in swissindex  #{@data_swissindex[gtin][:atc_code]}"
+        elsif atc_code.length > @data_swissindex[gtin][:atc_code].length
+          shorter_in_swissindex += 1
+          Util.debug_msg "SwissIndex #{gtin}: ATC code #{atc_code} shorter in swissindex  #{@data_swissindex[gtin][:atc_code]}"
+        else
+          different_atc_in_swissindex += 1
+          Util.debug_msg "SwissIndex #{gtin}: ATC code #{atc_code} differs from swissindex  #{@data_swissindex[gtin][:atc_code]}"
+        end
+
+        if not   @data_swissmedic[gtin]
+          Util.debug_msg "#{gtin}: Not in SwissMedic #{item}"
+          not_in_swissmedic += 1
+        elsif atc_code == @data_swissmedic[gtin][:atc_code]
+          Util.debug_msg "SwissMedic #{gtin}: ATC code #{atc_code} matches swissmedic  #{@data_swissmedic[gtin][:atc_code]}"
+          match_in_swissmedic += 1
+        elsif atc_code.length < @data_swissmedic[gtin][:atc_code].length
+          longer_in_swissmedic += 1
+          Util.debug_msg "SwissMedic #{gtin}: ATC code #{atc_code} longer  in swissmedic  #{@data_swissmedic[gtin][:atc_code]}"
+        elsif atc_code.length > @data_swissmedic[gtin][:atc_code].length
+          shorter_in_swissmedic += 1
+          Util.debug_msg "SwissMedic #{gtin}: ATC code #{atc_code} shorter in swissmedic  #{@data_swissmedic[gtin][:atc_code]}"
+        else
+          different_atc_in_swissmedic += 1
+          Util.debug_msg "SwissMedic #{gtin}: ATC code #{atc_code} differs from swissmedic  #{@data_swissmedic[gtin][:atc_code]}"
+        end
+        total1 = not_in_swissindex + match_in_swissindex + longer_in_swissindex +  shorter_in_swissindex + different_atc_in_swissindex
+        total2 = not_in_swissmedic + match_in_swissmedic + longer_in_swissmedic +  shorter_in_swissmedic + different_atc_in_swissmedic
+        # binding.pry if j != (total1 + matching_atc_codes)
+        # binding.pry if j != (total2 + matching_atc_codes)
+        # Util.debug_msg "#{gtin}: j #{j} finished #{total1} #{total2} #{atc_code} matching_atc_codes #{matching_atc_codes}"
+      }
+      Util.info  "Result of verifing data from BAG (SL):
+  BAG-data fetched from #{@bag.origin}.
+  BAG had #{@data_bag.size} entries
+  #{@bag_entries_without_gtin.size} entries had no GTIN field
+  Not in SwissMedic #{not_in_swissmedic}
+  Not in SwissIndex #{not_in_swissindex}
+  Comparing ATC-Codes between BAG and Swissmedic
+    #{sprintf("%6d", matching_atc_codes)} items had the same ATC code in BAG, SwissIndex and SwissMedic
+    #{sprintf("%6d", match_in_swissindex)} are the same in SwissMedic and BAG
+    #{sprintf("%6d", different_atc_in_swissmedic)} are different in SwissMedic and BAG
+    #{sprintf("%6d", shorter_in_swissmedic)} are shorter in SwissMedic than in BAG
+    #{sprintf("%6d", longer_in_swissindex)} are longer in SwissMedic than in BAG
+  Comparing ATC-Codes between BAG and Swissindex
+    #{sprintf("%6d", matching_atc_codes)} items had the same ATC code in BAG, SwissIndex and SwissMedic
+    #{sprintf("%6d", match_in_swissindex)} are the same in SwissIndex and BAG
+    #{sprintf("%6d", different_atc_in_swissindex)} are different in SwissMedic and BAG
+    #{sprintf("%6d", shorter_in_swissindex)} are shorter in SwissIndex than in BAG
+    #{sprintf("%6d", longer_in_swissindex)} are longer in SwissIndex than in BAG
+"
+    end
+
+    def check_swissmedic
+      matching = 0
+      not_in_bag = 0
+      not_in_swissindex = 0
+      matching_atc_codes = 0
+      shorter_in_swissmedic = 0
+      longer_in_swissindex = 0
+      different_atc = 0
+      @data_swissmedic.each{
+        |gtin, item|
+        if @data_bag[gtin] and @data_swissindex[gtin] and @data_bag[gtin][1] == @data_swissindex[gtin][1]
+          matching += 1
+          next
+        end
+        unless @data_swissindex[gtin]
+          Util.debug_msg "#{gtin}: Not in SwissIndex #{item}"
+          not_in_swissindex += 1
+          next
+        end
+        if item[:atc_code] == @data_swissindex[gtin][:atc_code]
+          Util.debug_msg "SwissIndex #{gtin}: ATC code #{item[:atc_code]} matches swissindex  #{@data_swissindex[gtin][:atc_code]}"
+          matching_atc_codes += 1
+        elsif item[:atc_code].length < @data_swissindex[gtin][:atc_code].length
+          longer_in_swissindex += 1
+          Util.debug_msg "SwissIndex #{gtin}: ATC code #{item[:atc_code]} longer  in swissindex  #{@data_swissindex[gtin][:atc_code]}"
+        elsif item[:atc_code].length > @data_swissindex[gtin][:atc_code].length
+          shorter_in_swissmedic += 1
+          Util.debug_msg "SwissIndex #{gtin}: ATC code #{item[:atc_code]} shorter in swissindex  #{@data_swissindex[gtin][:atc_code]}"
+        else
+          different_atc += 1
+          Util.debug_msg "SwissIndex #{gtin}: ATC code #{item[:atc_code]} differs from swissindex  #{@data_swissindex[gtin][:atc_code]}"
+        end
+        unless @data_bag[gtin]
+          Util.debug_msg "#{gtin}: Not in BAG #{item}"
+          not_in_bag += 1
+          next
+        end
+      }
+      Util.info  "Result of verifing data from swissmedic:
+  SwissMedic had #{@data_swissmedic.size} entries. Fetched from #{@swissmedic.origin}
+  SwissIndex #{@data_swissindex.size} entries. Fetched from #{@swissindex.origin}
+  BAG #{@data_bag.size} entries. #{@bag_entries_without_gtin.size} entries had no GTIN field. Fetched from #{@bag.origin}
+  Matching #{matching} items.
+  Not in BAG #{not_in_bag}
+  Not in SwissIndex #{not_in_swissindex}
+  Comparing ATC-Codes between Swissmedic and Swissindex
+    #{sprintf("%6d", matching_atc_codes)} match
+    #{sprintf("%6d", different_atc)} are different
+    #{sprintf("%6d", matching_atc_codes)} are the same in SwissIndex and SwissMedic
+    #{sprintf("%6d", shorter_in_swissmedic)} are shorter in SwissIndex
+    #{sprintf("%6d", longer_in_swissindex)} are longer in SwissIndex
+"
+    end
+
+    def compare
+      all_gtin = @data_bag.merge(@data_swissindex).merge(@data_swissmedic).sort
+      matching_atc_codes = 0
+      not_in_bag = 0
+      not_in_swissmedic = 0
+      not_in_swissindex = 0
+      different_atc = 0
+      all_gtin.each{
+        |gtin, item|
+        if @data_bag[gtin] and @data_swissindex[gtin] and @data_swissmedic[gtin] and
+          @data_bag[gtin][:atc_code] == @data_swissindex[gtin][:atc_code] and
+          @data_bag[gtin][:atc_code] == @data_swissindex[gtin][:atc_code]
+          matching_atc_codes += 1
+          next
+        end
+        unless @data_swissmedic[gtin]
+          Util.debug_msg "#{gtin}: Not in SwissMedic #{item}"
+          not_in_swissmedic += 1
+          next
+        end
+        unless @data_swissindex[gtin]
+          Util.debug_msg "#{gtin}: Not in SwissIndex #{item}"
+          not_in_swissindex += 1
+          next
+        end
+        unless @data_bag[gtin]
+          Util.debug_msg "#{gtin}: Not in BAG #{item}"
+          not_in_bag += 1
+          next
+        end
+        different_atc += 1
+        Util.debug_msg "#{gtin}: ATC code differs BAG #{@data_bag[gtin][:atc_code]} swissindex  #{@data_swissindex[gtin][:atc_code]}"
+      }
+      Util.info  "Comparing all GTIN-codes:
+  Found infos about #{all_gtin.size} entries
+  BAG #{@data_bag.size} entries. #{@bag_entries_without_gtin.size} entries had no GTIN field. Fetched from #{@bag.origin}
+  SwissIndex #{@data_swissindex.size} entries. Fetched from #{@swissindex.origin}
+  SwissMedic #{@data_swissmedic.size} entries. Fetched from #{@swissmedic.origin}
+  #{sprintf("%6d", matching_atc_codes)} items had the same ATC code in BAG, SwissIndex and SwissMedic
+  #{sprintf("%6d", not_in_bag)} not in BAG
+  #{sprintf("%6d", not_in_swissindex)} not in SwissIndex
+  #{sprintf("%6d", not_in_swissmedic)} not in SwissMedic
+  #{sprintf("%6d", different_atc)} ATC-Codes differed
+"
+    end
+  end
+  class Swissmedic
+    def Swissmedic.get_latest
+      Util.debug_msg 'test'
+      @index_url = 'https://www.swissmedic.ch/arzneimittel/00156/00221/00222/00230/index.html?lang=de'
+      Util.debug_msg("SwissmedicPlugin @index_url #{@index_url}")
+      latest_name, target =  Util.get_latest_and_dated_name('Packungen', '.xlsx')
+      if File.exist?(target)
+        Util.debug_msg "#{__FILE__}: #{__LINE__} skip writing #{target} as it already exists and is #{File.size(target)} bytes."
+        return target
+      end
+      Util.debug_msg "target #{target} #{latest_name}"
+      latest = ''
+      if(File.exist? latest_name)
+        latest = File.read latest_name
+        return latest_name
+      end
+
+      agent=Mechanize.new
+      page = agent.get @index_url
+      links = page.links.select do |link|
+        /Packungen/iu.match link.attributes['title']
+      end
+      link = links.first or raise "could not identify url to Packungen.xlsx"
+      file = agent.get(link.href)
+      download = file.body
+
+      if(download[-1] != ?\n)
+        download << "\n"
+      end
+      if(!File.exist?(latest_name) or download.size != File.size(latest_name))
+        File.open(target, 'w') { |fh| fh.puts(download) }
+        msg = "#{__FILE__}: #{__LINE__} updated download.size is #{download.size} -> #{target} #{File.size(target)}"
+        msg += "#{target} now #{File.size(target)} bytes != #{latest_name} #{File.size(latest_name)}" if File.exists?(latest_name)
+        Util.debug_msg(msg)
+        target
+      else
+        Util.debug_msg "#{__FILE__}: #{__LINE__} skip writing #{target} as #{latest_name} is #{File.size(latest_name)} bytes. Returning latest"
+        nil
+      end
+    end
+
+  end
+end