oddb2xml 2.1.7 → 2.1.8

data/lib/oddb2xml/cli.rb

@@ -13,17 +13,17 @@ require 'date' # for today
 module Oddb2xml
   
   class Cli
+    attr_reader :options
     SUBJECTS  = %w[product article]
     ADDITIONS = %w[substance limitation interaction code]
     OPTIONALS = %w[fi fi_product]
-    LANGUAGES = %w[DE FR] # EN does not exist
     def initialize(args)
       @options = args
       Oddb2xml.save_options(@options)
       @mutex = Mutex.new
       # product
-      @items = {} # Items from Preparations.xml in BAG
-      @index = {} # Base index from swissINDEX
+      @items = {} # Items from Preparations.xml in BAG, using GTINs as key
+      @refdata_types = {} # Base index from refdata
       @flags = {} # narcotics flag files repo
       @lppvs = {} # lppv.txt from files repo
       @infos = {} # [option] FI from SwissmedicInfo
@@ -36,9 +36,6 @@ module Oddb2xml
       # addres
       @companies = [] # betrieb
       @people    = [] # medizinalperson
-      LANGUAGES.each do |lang|
-        @index[lang] = {}
-      end
       @_message = false
     end
     def run
@@ -69,14 +66,12 @@ module Oddb2xml
         threads << download(:bm_update) # oddb2xml_files
         threads << download(:lppv) # oddb2xml_files
         threads << download(:bag) # bag.e-mediat
-        LANGUAGES.each do |lang|
-          types.each do |type|
-            threads << download(:index, type, lang) # swissindex
-          end
+        types.each do |type|
+          threads << download(:refdata, type) # refdata
         end
       end
       begin
-        threads.map(&:join)
+        # threads.map(&:join) # TODO
       rescue SystemExit
         @mutex.synchronize do
           if @_message
@@ -95,6 +90,7 @@ module Oddb2xml
     private
     def build
       Oddb2xml.log("Start build")
+      puts "#{File.basename(__FILE__)}:#{__LINE__}: @refdata_types.keys #{@refdata_types.keys}"
       begin
         # require 'pry'; binding.pry
         @_files = {"calc"=>"oddb_calc.xml"} if @options[:calc] and not @options[:extended]
@@ -109,14 +105,11 @@ module Oddb2xml
               builder.people    = @people
             else # product
               if @options[:format] != :dat
-                index = {}
-                LANGUAGES.each do |lang|
-                  index[lang] = {} unless index[lang]
-                  types.each do |type|
-                    index[lang].merge!(@index[lang][type]) if @index[lang][type]
-                  end
+                refdata = {}
+                types.each do |type|
+                  refdata.merge!(@refdata_types[type]) if @refdata_types[type]
                 end
-                builder.index = index
+                builder.refdata = refdata
                 builder.subject = sbj
               end
               # common sources
@@ -136,12 +129,9 @@ module Oddb2xml
           output = ''
           if !@options[:address] and (@options[:format] == :dat)
             types.each do |type|
-              index = {}
-              LANGUAGES.each do |lang|
-                index[lang] = @index[lang][type]
-              end
+              refdata1 = {}
               _sbj = (type == :pharma ? :dat : :with_migel_dat)
-              builder.index   = index
+              builder.refdata = @refdata_types[type]
               builder.subject = _sbj
               builder.ean14   = @options[:ean14]
               if type == :nonpharma
@@ -166,11 +156,11 @@ module Oddb2xml
         raise Interrupt
       end
     end
-    def download(what, type=nil, lang=nil)
+    def download(what, type=nil)
       case what
       when :company, :person
         var = (what == :company ? 'companies' : 'people')
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = MedregbmDownloader.new(what)
           str = downloader.download
           self.instance_variable_set(
@@ -179,7 +169,7 @@ module Oddb2xml
           )
         end
       when :fachinfo
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = SwissmedicInfoDownloader.new
           xml = downloader.download
           @mutex.synchronize do
@@ -190,7 +180,7 @@ module Oddb2xml
         end
       when :orphan, :fridge
         var = what.to_s + 's'
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = SwissmedicDownloader.new(what)
           bin = downloader.download
           self.instance_variable_set(
@@ -200,7 +190,7 @@ module Oddb2xml
 #          Oddb2xml.log("SwissmedicExtractor added #{self.instance_variable_get("@#{var}".intern).size} #{var}. File #{bin} was #{File.size(bin)} bytes")
         end
       when :interaction
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = EphaDownloader.new
           str = downloader.download
           @mutex.synchronize do
@@ -209,7 +199,7 @@ module Oddb2xml
           end
         end
       when :migel
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = MigelDownloader.new
           bin = downloader.download
           @mutex.synchronize do
@@ -218,7 +208,7 @@ module Oddb2xml
           end
         end
       when :package
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = SwissmedicDownloader.new(:package, @options)
           bin = downloader.download
           @mutex.synchronize do
@@ -228,7 +218,7 @@ module Oddb2xml
           end
         end
       when :bm_update
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = BMUpdateDownloader.new
           str = downloader.download
           @mutex.synchronize do
@@ -237,7 +227,7 @@ module Oddb2xml
           end
         end
       when :lppv
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = LppvDownloader.new
           str = downloader.download
           @mutex.synchronize do
@@ -246,31 +236,33 @@ module Oddb2xml
           end
         end
       when :bag
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = BagXmlDownloader.new(@options)
           xml = downloader.download
           @mutex.synchronize do
             hsh = BagXmlExtractor.new(xml).to_hash
             @items = hsh
-            Oddb2xml.log("BagXmlDownloader added #{@items.size} items")
+            Oddb2xml.log("BagXmlDownloader added #{@items.size} items. #{@items.keys}")
           end
         end
       when :zurrose
-        Thread.new do
+        begin # instead of Thread.new do
           downloader = ZurroseDownloader.new(@options, @options[:transfer_dat])
           xml = downloader.download
           Oddb2xml.log("zurrose xml #{xml.size} bytes")
           @mutex.synchronize do
             hsh = ZurroseExtractor.new(xml, @options[:extended]).to_hash
+            Oddb2xml.log("zurrose added #{hsh.size} items from xml with #{xml.size} bytes")
             @infos_zur_rose = hsh
-            Oddb2xml.log("zurrose added #{@infos_zur_rose.size} items from xml with #{xml.size} bytes")
           end
         end
-      when :index
-        Thread.new do
-          downloader = SwissIndexDownloader.new(@options, type, lang)
+      when :refdata
+        begin # instead of Thread.new do
+          downloader = RefdataDownloader.new(@options, type)
           begin
             xml = downloader.download
+            Oddb2xml.log("refdata #{type} xml #{xml.size} bytes")
+            xml
           rescue SystemExit
             @mutex.synchronize do
               unless @_message # hook only one exit
@@ -280,8 +272,9 @@ module Oddb2xml
             end
           end
           @mutex.synchronize do
-            hsh = SwissIndexExtractor.new(xml, type).to_hash
-            @index[lang][type] = hsh
+            hsh = RefdataExtractor.new(xml, type).to_hash
+            @refdata_types[type] = hsh
+            Oddb2xml.log("refdata #{type} added #{hsh.size} keys now #{@refdata_types.keys} items from xml with #{xml.size} bytes")
           end
         end
       end
@@ -335,19 +328,16 @@ module Oddb2xml
         lines << ParseComposition.report
       end
       unless @options[:address]
-        LANGUAGES.each do |lang|
-          lines << lang
-          types.each do |type|
-            if @index[lang][type]
-              indices = @index[lang][type].values.flatten.length
-              if type == :nonpharma
-                migel_xls  = @migel.values.compact.select{|m| !m[:pharmacode].empty? }.map{|m| m[:pharmacode] }
-                nonpharmas = @index[lang][type].keys
-                indices += (migel_xls - nonpharmas).length # ignore duplicates, null
-                lines << sprintf("\tNonPharma products: %i", indices)
-              else
-                lines << sprintf("\tPharma products: %i", indices)
-              end
+        types.each do |type|
+          if @refdata_types[type]
+            indices = @refdata_types[type].values.flatten.length
+            if type == :nonpharma
+              migel_xls  = @migel.values.compact.select{|m| !m[:pharmacode].empty? }.map{|m| m[:pharmacode] }
+              nonpharmas = @refdata_types[type].keys
+              indices += (migel_xls - nonpharmas).length # ignore duplicates, null
+              lines << sprintf("\tNonPharma products: %i", indices)
+            else
+              lines << sprintf("\tPharma products: %i", indices)
             end
           end
         end
@@ -365,7 +355,7 @@ module Oddb2xml
       end
       puts lines.join("\n")
     end
-    def types # swissindex
+    def types # RefData
       @_types ||=
         if @options[:nonpharma]
           [:pharma, :nonpharma]

data/lib/oddb2xml/downloader.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+require 'net/ntlm/version' # needed to avoid error: uninitialized constant Net::NTLM::VERSION
+require 'rubyntlm'
 require 'mechanize'
 require 'zip'
 require 'savon'
@@ -34,7 +36,7 @@ module Oddb2xml
     end
   end
   class Downloader
-    attr_reader :type
+    attr_reader :type, :agent
     def initialize(options={}, url=nil)
       @options     = options
       @url         = url
@@ -55,6 +57,7 @@ module Oddb2xml
         cert_store.add_file(File.expand_path('../../../tools/cacert.pem', __FILE__))
         @agent.cert_store = cert_store
       end
+      @agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
     protected
     def retrievable?
@@ -133,14 +136,14 @@ module Oddb2xml
   class BMUpdateDownloader < Downloader
     include DownloadMethod
     def download
-      @url ||= 'https://raw.github.com/zdavatz/oddb2xml_files/master/BM_Update.txt'
+      @url ||= 'https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/BM_Update.txt'
       download_as('oddb2xml_files_bm_update.txt', 'r')
     end
   end
   class LppvDownloader < Downloader
     include DownloadMethod
     def download
-      @url ||= 'https://raw.github.com/zdavatz/oddb2xml_files/master/LPPV.txt'
+      @url ||= 'https://raw.githubusercontent.com/zdavatz/oddb2xml_files/master/LPPV.txt'
       download_as('oddb2xml_files_lppv.txt', 'r')
     end
   end
@@ -203,11 +206,10 @@ module Oddb2xml
       content
     end
   end
-  class SwissIndexDownloader < Downloader
-    def initialize(options={}, type=:pharma, lang='DE')
+  class RefdataDownloader < Downloader
+    def initialize(options={}, type=:pharma)
       @type = (type == :pharma ? 'Pharma' : 'NonPharma')
-      @lang = lang
-      url = "https://index.ws.e-mediat.net/Swissindex/#{@type}/ws_#{@type}_V101.asmx?WSDL"
+      url = "http://refdatabase.refdata.ch/Service/Article.asmx?WSDL"
       super(options, url)
     end
     def init
@@ -222,19 +224,21 @@ module Oddb2xml
     end
     def download
       begin
-        filename =  "swissindex_#{@type}_#{@lang}.xml"
-        file2save = File.join(Downloads, "swissindex_#{@type}_#{@lang}.xml")
+        filename =  "refdata_#{@type}.xml"
+        file2save = File.join(Downloads, "refdata_#{@type}.xml")
+				soap = %(<?xml version="1.0" encoding="UTF-8"?>
+<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns1="http://refdatabase.refdata.ch/Article_in" xmlns:ns2="http://refdatabase.refdata.ch/">
+  <SOAP-ENV:Body>
+    <ns2:DownloadArticleInput>
+      <ns1:ATYPE>#{@type.upcase}</ns1:ATYPE>
+    </ns2:DownloadArticleInput>
+  </SOAP-ENV:Body>
+  </SOAP-ENV:Envelope>
+</ns1:ATYPE></ns2:DownloadArticleInput></SOAP-ENV:Body>
+)
         return IO.read(file2save) if Oddb2xml.skip_download? and File.exists?(file2save)
         FileUtils.rm_f(file2save, :verbose => false)
-        soap = <<XML
-<?xml version="1.0" encoding="utf-8"?>
-<soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
-<soap:Body>
-  <lang xmlns="http://swissindex.e-mediat.net/Swissindex#{@type}_out_V101">#{@lang}</lang>
-</soap:Body>
-</soap:Envelope>
-XML
-        response = @client.call(:download_all, :xml => soap)
+        response = @client.call(:download, :xml => soap)
         if response.success?
           if xml = response.to_xml
             response = nil # win
@@ -282,7 +286,7 @@ XML
       begin
         FileUtils.rm(File.expand_path(file), :verbose => !defined?(RSpec)) if File.exists?(File.expand_path(file))
         page = @agent.get(@url)
-        if link_node = page.search(@xpath).first
+        if !page.class.is_a?(String) and link_node = page.search(@xpath).first
           link = Mechanize::Page::Link.new(link_node, @agent, page)
           response = link.click
           response.save_as(file)

data/lib/oddb2xml/extractor.rb

@@ -8,7 +8,6 @@ require 'csv'
 require 'oddb2xml/xml_definitions'
 
 module Oddb2xml
-  Strip_For_Sax_Machine = '<?xml version="1.0" encoding="utf-8"?>'+"\n"
   module TxtExtractorMethods
     def initialize(str)
       @io = StringIO.new(str)
@@ -83,8 +82,9 @@ module Oddb2xml
           phar = pac.Pharmacode
           phar = correct_code(phar.to_s, 7)
           ean = pac.GTIN
+          binding.pry if pac.GTIN.to_s.eql? '7680324750190'  # lansoyl
           search_key = phar.to_i != 0 ? phar : ean
-          # as common key with swissINDEX
+          # as common key with RefData Pharma/NonPharma data
           item[:pharmacodes] << phar
           # packages
           exf = {:price => '', :valid_date => '', :price_code => ''}
@@ -178,38 +178,34 @@ module Oddb2xml
     end
   end
 
-  class SwissIndexExtractor < Extractor
+  class RefdataExtractor < Extractor
     def initialize(xml, type)
       @type = (type == :pharma ? 'PHARMA' : 'NONPHARMA')
       super(xml)
     end
     def to_hash
       data = {}
-      result = PharmaEntry.parse(@xml.sub(Strip_For_Sax_Machine, ''), :lazy => true)
-      items = result.PHARMA.ITEM
+      result = SwissRegArticleEntry.parse(@xml.sub(Strip_For_Sax_Machine, ''), :lazy => true)
+      items = result.ARTICLE.ITEM
       items.each do |pac|
         item = {}
         item[:refdata]         = true
-        item[:_type]           = @type.downcase.intern
+        item[:_type]           = (typ  = pac.ATYPE.downcase.to_sym)  ? typ: ''
         item[:ean]             = (gtin = pac.GTIN)   ? gtin: ''
         item[:pharmacode]      = (phar = pac.PHAR)   ? phar: ''
-        item[:stat_date]       = (date = pac.SDATE)  ? date: ''
-        item[:lang]            = (lang = pac.LANG)   ? lang: ''
-        item[:desc]            = (dscr = pac.DSCR)   ? dscr: ''
+        item[:last_change]     = (date = Time.parse(pac.DT).to_s)  ? date: ''  # Date and time of last data change
+        item[:desc_de]         = (dscr = pac.NAME_DE)   ? dscr: ''
+        item[:desc_fr]         = (dscr = pac.NAME_FR)   ? dscr: ''
         item[:atc_code]        = (code = pac.ATC)    ? code.to_s : ''
-        # as quantity text
-        item[:additional_desc] = (dscr = pac.ADDSCR) ? dscr: ''
-        if comp = pac.COMP
-          item[:company_name] = (nam = comp.NAME) ? nam: ''
-          item[:company_ean]  = (gln = comp.GLN)  ? gln: ''
-        end
+				item[:company_name] = (nam = pac.AUTH_HOLDER_NAME) ? nam: ''
+				item[:company_ean]  = (gln = pac.AUTH_HOLDER_GLN)  ? gln: ''
         unless item[:pharmacode].empty?
           item[:pharmacode] = correct_code(item[:pharmacode].to_s, 7)
           unless data[item[:pharmacode]] # pharmacode => GTINs
-            data[item[:pharmacode]] = []
+            data[item[:ean]] = []
           end
-          data[item[:pharmacode]] << item
         end
+        data[item[:ean]] = item
       end
       data
     end
@@ -327,13 +323,15 @@ module Oddb2xml
       @sheet.each_with_index do |row, i|
         next if i.zero?
         phar = correct_code(row[1].to_s.gsub(/[^0-9]/, ''), 7)
-        data[phar] = {
+        ean = row[0].to_i.to_s
+        ean = '9999'+phar+'99' unless ean.length == 13
+        data[ean] = {
           :refdata         => true,
-          :ean             => row[0].to_i.to_s,
+          :ean             => ean,
           :pharmacode      => phar,
           :desc_de         => row[3],
           :desc_fr         => row[4],
-          :additional_desc => row[5], # quantity
+          :quantity        => row[5], # quantity
           :company_name    => row[6],
           :company_ean     => row[7].to_i.to_s,
         }
@@ -497,7 +495,7 @@ module Oddb2xml
           :ean   => ean13,
           :vat   => line[96],
           :description => line[10..59].sub(/\s+$/, ''),
-          :additional_desc => '',
+          :quantity => '',
           :pharmacode => pharma_code,
           :price => sprintf("%.2f", line[60,6].gsub(/(\d{2})$/, '.\1').to_f),
           :pub_price => sprintf("%.2f", line[66,6].gsub(/(\d{2})$/, '.\1').to_f),

data/lib/oddb2xml/util.rb

@@ -33,7 +33,7 @@ module Oddb2xml
   end
 
   def Oddb2xml.log(msg)
-    return unless @options[:log]
+    # return unless @options[:log]
     $stdout.puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}: #{msg}"
     $stdout.flush
   end
@@ -47,6 +47,7 @@ module Oddb2xml
   end
   
   def Oddb2xml.skip_download(file)
+    return false if defined?(VCR)
     dest = "#{Downloads}/#{File.basename(file)}"
     if File.exists?(dest)
       FileUtils.cp(dest, file, :verbose => false, :preserve => true) unless File.expand_path(file).eql?(dest)

data/lib/oddb2xml/version.rb

@@ -1,3 +1,3 @@
 module Oddb2xml
-  VERSION = "2.1.7"
+  VERSION = "2.1.8"
 end

data/lib/oddb2xml/xml_definitions.rb

@@ -3,6 +3,8 @@
 
 require 'sax-machine'
 
+Strip_For_Sax_Machine = '<?xml version="1.0" encoding="utf-8"?>'+"\n"
+
 class PriceElement
   include SAXMachine
   element :Price
@@ -210,18 +212,6 @@ class ItemContent
   element :COMP, :class => CompElement
 end
 
-class PharmaContent
-  include SAXMachine
-  attribute :CREATION_DATETIME
-  elements :ITEM, :class => ItemContent
-end
-
-class PharmaEntry
-  include SAXMachine
-  element :CREATION_DATETIME
-  element :PHARMA, :class => PharmaContent
-end
-
 class MedicalInformationContent
   include SAXMachine
   attribute :type
@@ -230,6 +220,7 @@ class MedicalInformationContent
   element :title
   element :authHolder
   element :authNrs
+  element :authNrs
   element :style
   element :content
 end
@@ -249,3 +240,29 @@ class MedicalInformationsEntry
   element :medicalInformations, :class => MedicalInformationsContent
 end
 
+class SwissRegItemContentContent
+  include SAXMachine
+  attribute :DT
+  element :ATYPE
+  element :GTIN
+  element :PHAR
+  element :SWMC_AUTHNR
+  element :NAME_DE
+  element :NAME_FR
+  element :ADDSCR
+  element :ATC
+  element :AUTH_HOLDER_NAME
+  element :AUTH_HOLDER_GLN
+end
+
+class SwissRegArticleContent
+  include SAXMachine
+  attribute :CREATION_DATETIME
+  elements :ITEM, :class => SwissRegItemContentContent
+end
+
+class SwissRegArticleEntry
+  include SAXMachine
+  element :CREATION_DATETIME
+  element :ARTICLE, :class => SwissRegArticleContent
+end

data/oddb2xml.gemspec

@@ -20,20 +20,24 @@ Gem::Specification.new do |spec|
   # We fix the version of the spec to newer versions only in the third position
   # hoping that these version fix only security/severe bugs
   # Consulted the Gemfile.lock to get 
-  spec.add_dependency 'rubyzip', '~> 1.1.3'
-  spec.add_dependency 'archive-tar-minitar', '~> 0.5.2'
-  spec.add_dependency 'mechanize', '~> 2.5.1'
-  spec.add_dependency 'nokogiri'
-  spec.add_dependency 'savon', '~> 2.4.0'
-  spec.add_dependency 'spreadsheet', '~> 1.0.0'
-  spec.add_dependency 'rubyXL', '~> 3.3.1'
-  spec.add_dependency 'sax-machine',  '~> 0.1.0'
-  spec.add_dependency 'parslet', '~> 1.7.0'
-  
+  spec.add_dependency 'rubyzip'#, '~> 1.1.3'
+  spec.add_dependency 'archive-tar-minitar'#, '~> 0.5.2'
+  spec.add_dependency 'mechanize'#, '~> 2.5.1'
+  spec.add_dependency 'nokogiri'#, '~> 1.6.0'
+  spec.add_dependency 'savon'#, '~> 2.11.0'
+  spec.add_dependency 'spreadsheet'#, '~> 1.0.0'
+  spec.add_dependency 'rubyXL'#, '~> 3.3.1'
+  spec.add_dependency 'sax-machine'#,  '~> 0.1.0'
+  spec.add_dependency 'parslet'#, '~> 1.7.0'
+  spec.add_dependency 'rubyntlm'#, '>= 0.3.2'
+  spec.add_dependency 'multi_json'#, '>= 0.3.2'
+
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "rdoc"
+  spec.add_development_dependency "vcr"
+  spec.add_development_dependency "timecop"
 end