oddb2xml 1.6.2 → 1.6.3

data/Gemfile

@@ -14,3 +14,7 @@ group :development do
   gem 'hoe', '>= 3.4'
   gem 'rdoc'
 end
+
+group :debugger do
+  gem 'pry-debugger'
+end

data/History.txt

@@ -1,3 +1,9 @@
+=== 1.6.3 / 10.12.2013
+
+* Added more specs. Use ean to find price-info if no pharmacode
+* Fix running specs
+* Added skip-download for debugging
+
 === 1.6.2 / 09.12.2013
 
 * Read article with no pharmacode from BAG.xml Preparation.xml

data/bin/oddb2xml

@@ -13,6 +13,7 @@ def help
 #$0 ver.#{Oddb2xml::VERSION}
 Usage:
   oddb2xml [option]
+    produced files are found under data
     -a T, --append=T     Additional target. T, only 'nonpharma' is available.
     -c F, --compress=F   Compress format F. {tar.gz|zip}
     -f F, --format=F     File format F, default is xml. {xml|dat}
@@ -24,6 +25,10 @@ Usage:
     -t S, --tag-suffix=S XML tag suffix S. Default is none. [A-z0-9]
                          If S is given, it is also used as prefix of filename.
     -x N, --context=N    context N {product|address}. product is default.
+                         
+                         For debugging purposes
+    --skip-download      skips downloading files it the file is already under data/downloads.
+                         Downloaded files are saved under data/downloads
     -h,   --help         Show this help message.
 EOS
 end
@@ -38,6 +43,7 @@ opts = {
   :tag_suffix   => nil,
   :debug        => false,
   :ean14        => false,
+  :skip_download=> false,
 }
 
 parser.on('-a v', '--append v',     /^nonpharma$/)  {|v| opts[:nonpharma] = true }
@@ -48,7 +54,7 @@ parser.on('-i v', '--include v',    /^ean14$/)      {|v| opts[:ean14] = true }
 parser.on('-t v', '--tag-suffix v', /^[A-z0-9]*$/i) {|v| opts[:tag_suffix] = v.upcase }
 parser.on('-x v', '--context v',    /^addr(ess)*$/i){|v| opts[:address] = true }
 parser.on('-p v', '--price v',      /^zurrose$/)    {|v| opts[:price] = v.intern }
-#parser.on('--debug') {|v| opts[:debug] = true }
+parser.on('--skip-download') {|v| opts[:skip_download] = true }
 parser.on_tail('-h', '--help') { puts help; exit }
 
 args = ARGV.dup
@@ -81,6 +87,6 @@ then you will not see anymore warnings
   end
   ui.run
 rescue Interrupt
-  puts
   exit
 end
+puts "#{Time.now}: #{__LINE__} done"; $stdout.sync

data/lib/oddb2xml/builder.rb

@@ -22,7 +22,8 @@ module Oddb2xml
                   :infos, :packs, :prices,
                   :ean14, :tag_suffix,
                   :companies, :people
-    def initialize
+    def initialize(args = {})
+      @options    = args
       @subject    = nil
       @index      = {}
       @items      = {}
@@ -390,8 +391,12 @@ module Oddb2xml
             seq = obj[:seq]
             length += 1
             xml.PRD('DT' => '') {
-              xml.GTIN obj[:ean].to_s
-              ppac = ((_ppac = @packs[obj[:ean].to_s[4..11].intern] and !_ppac[:is_tier]) ? _ppac : {})
+              ean = obj[:ean].to_s
+              xml.GTIN ean
+              ppac = ((_ppac = @packs[ean[4..11].intern] and !_ppac[:is_tier]) ? _ppac : {})
+              unless ppac
+                 ppac = @packs.find{|pac| pac.ean == ean }.first               
+              end
               xml.PRODNO ppac[:prodno] if ppac[:prodno] and !ppac[:prodno].empty?
               if seq
                 %w[de fr].each do |l|
@@ -537,8 +542,12 @@ module Oddb2xml
               fr_idx = obj[:fr][i]              # swissindex FR
               pac,no8 = nil,de_idx[:ean][4..11] # BAG-XML(SL/LS)
               ppac = nil                        # Packungen
+              ean = de_idx[:ean]
               if obj[:seq]
                 pac = obj[:seq][:packages][de_idx[:pharmacode]]
+                pac = obj[:seq][:packages][ean] unless pac
+              else
+                pac = @items[ean][:packages][ean] if @items and @items[ean]
               end
               if no8
                 ppac = ((_ppac = @packs[no8.intern] and !_ppac[:is_tier]) ? _ppac : nil)
@@ -573,7 +582,7 @@ module Oddb2xml
                 if de_idx
                   xml.SALECD(de_idx[:status].empty? ? 'N' : de_idx[:status])
                 end
-                if pac
+                if pac and pac[:limitation_points]
                   #xml.INSLIM
                   xml.LIMPTS pac[:limitation_points] unless pac[:limitation_points].empty?
                 end
@@ -656,7 +665,7 @@ module Oddb2xml
                   #xml.LINENO
                   #xml.NOUNITS
                 #}
-                if pac
+                if pac and pac[:prices]
                   pac[:prices].each_pair do |key, price|
                     xml.ARTPRI {
                      xml.VDAT  price[:valid_date] unless price[:valid_date].empty?
@@ -956,6 +965,7 @@ module Oddb2xml
             pac,no8 = nil,nil
             if obj[:seq]
               pac = obj[:seq][:packages][idx[:pharmacode]]
+              pac = obj[:seq][:packages][idx[:ean]] unless pac
             end
             # :swissmedic_numbers
             if pac

data/lib/oddb2xml/cli.rb

@@ -5,6 +5,7 @@ require 'oddb2xml/builder'
 require 'oddb2xml/downloader'
 require 'oddb2xml/extractor'
 require 'oddb2xml/compressor'
+require 'oddb2xml/util'
 
 module Oddb2xml
   class Cli
@@ -14,6 +15,7 @@ module Oddb2xml
     LANGUAGES = %w[DE FR] # EN does not exist
     def initialize(args)
       @options = args
+      Oddb2xml.save_options(@options)
       @mutex = Mutex.new
       # product
       @items = {} # Items from Preparations.xml in BAG
@@ -88,7 +90,7 @@ module Oddb2xml
     def build
       begin
         files.each_pair do |sbj, file|
-          builder = Builder.new do |builder|
+          builder = Builder.new(@options) do |builder|
             if @options[:address]
               builder.subject   = sbj
               builder.companies = @companies
@@ -143,7 +145,7 @@ module Oddb2xml
       rescue Interrupt
         files.values.each do |file|
           if File.exist? file
-            File.unlink file
+            File.unlink(file) # we don't save it as it might be only partly downloaded
           end
         end
         raise Interrupt

data/lib/oddb2xml/compressor.rb

@@ -36,13 +36,11 @@ module Oddb2xml
         end
         if File.exists? @compress_file
           @contents.each do |file|
-            File.unlink file
+            Oddb2xml.download_finished(file)
           end
         end
       rescue Errno::ENOENT, StandardError
-        if File.exists? @compress_file
-          File.unlink @compress_file
-        end
+        Oddb2xml.download_finished(@compress_file)
         return false
       end
       return true

data/lib/oddb2xml/downloader.rb

@@ -9,18 +9,21 @@ module Oddb2xml
     private
     def download_as(file, option='r')
       data = nil
-      begin
-        response = @agent.get(@url)
-        response.save_as(file)
-        response = nil # win
+      if Oddb2xml.skip_download(file)
         io = File.open(file, option)
         data = io.read
-      rescue Timeout::Error, Errno::ETIMEDOUT
-        retrievable? ? retry : raise
-      ensure
-        io.close if io and !io.closed? # win
-        if File.exists?(file)
-          File.unlink(file)
+      else
+        begin
+          response = @agent.get(@url)
+          response.save_as(file)
+          response = nil # win
+          io = File.open(file, option)
+          data = io.read
+        rescue Timeout::Error, Errno::ETIMEDOUT
+          retrievable? ? retry : raise
+        ensure
+          io.close if io and !io.closed? # win
+          Oddb2xml.download_finished(file)
         end
       end
       return data
@@ -153,17 +156,17 @@ module Oddb2xml
         if @options[:debug]
           FileUtils.copy(File.expand_path("../../../spec/data/#{file}", __FILE__), '.')
         else
-          response = @agent.get(@url)
-          response.save_as(file)
-          response = nil # win
+          unless Oddb2xml.skip_download(file)
+            response = @agent.get(@url)
+            response.save_as(file)
+            response = nil # win
+          end
         end
-        return read_xml_form_zip(/^Preparation/iu, file)
+        read_xml_form_zip(/^Preparation/iu, file)
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       ensure
-        if File.exists?(file)
-          File.unlink(file)
-        end
+        Oddb2xml.download_finished(file)
       end
     end
   end
@@ -189,6 +192,7 @@ module Oddb2xml
         if @options[:debug]
           return File.read(File.expand_path("../../../spec/data/swissindex_#{@type}_#{@lang}.xml", __FILE__))
         end
+        file2save = File.expand_path("../../../data/download/swissindex_#{@type}_#{@lang}.xml", __FILE__)
         soap = <<XML
 <?xml version="1.0" encoding="utf-8"?>
 <soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/">
@@ -201,6 +205,7 @@ XML
         if response.success?
           if xml = response.to_xml
             response = nil # win
+            File.open(file2save, 'w+') { |file| file.puts xml }
             return xml
           else
             # received broken data or internal error
@@ -239,9 +244,11 @@ XML
         page = @agent.get(@url)
         if link_node = page.search(@xpath).first
           link = Mechanize::Page::Link.new(link_node, @agent, page)
-          response = link.click
-          response.save_as(file)
-          response = nil # win
+          unless Oddb2xml.skip_download(file)
+            response = link.click
+            response.save_as(file)
+            response = nil # win
+          end
         end
         io = File.open(file, 'rb')
         return io.read
@@ -249,9 +256,7 @@ XML
         retrievable? ? retry : raise
       ensure
         io.close if io and !io.closed?
-        if File.exists?(file)
-          File.unlink(file)
-        end
+        Oddb2xml.download_finished(file)
       end
     end
   end
@@ -264,37 +269,29 @@ XML
     def download
       file = "swissmedic_info.zip"
       begin
-        response = nil
-        if home = @agent.get(@url)
-          form = home.form_with(:id => 'Form1')
-          bttn = form.button_with(:name => 'ctl00$MainContent$btnOK')
-          if page = form.submit(bttn)
-            form = page.form_with(:id => 'Form1')
-            bttn = form.button_with(:name => 'ctl00$MainContent$BtnYes')
-            response = form.submit(bttn)
+        unless Oddb2xml.skip_download(file)
+          response = nil
+          if home = @agent.get(@url)
+            form = home.form_with(:id => 'Form1')
+            bttn = form.button_with(:name => 'ctl00$MainContent$btnOK')
+            if page = form.submit(bttn)
+              form = page.form_with(:id => 'Form1')
+              bttn = form.button_with(:name => 'ctl00$MainContent$BtnYes')
+              response = form.submit(bttn)
+            end
+          end
+          if response          
+            response.save_as(file)
+            response = nil # win
           end
         end
-        if response
-          response.save_as(file)
-          response = nil # win
-        end
-        return read_xml_form_zip(/^AipsDownload_/iu, file)
+        read_xml_form_zip(/^AipsDownload_/iu, file)
       rescue Timeout::Error, Errno::ETIMEDOUT
         retrievable? ? retry : raise
       rescue NoMethodError
         # pass
       ensure
-        # win
-        deleted = false
-        if File.exists?(file)
-          until deleted
-            begin
-              File.unlink(file)
-              deleted = true
-            rescue Errno::EACCES # Permission Denied on Windows
-            end
-          end
-        end
+        Oddb2xml.download_finished(file)
       end
     end
   end

data/lib/oddb2xml/extractor.rb

@@ -240,6 +240,7 @@ module Oddb2xml
           no8 = sprintf('%05d',row[i_5].to_i) + sprintf('%03d',row[i_3].to_i)
           prodno = sprintf('%05d',row[i_5].to_i) + row[p_1_2].to_i.to_s
           unless no8.empty?
+            next if no8.to_i == 0
             ean_base12 = "7680#{no8}"
             data[no8.intern] = {
               :ean                  => (ean_base12.ljust(12, '0') + calc_checksum(ean_base12)),

data/lib/oddb2xml/version.rb

@@ -1,3 +1,3 @@
 module Oddb2xml
-  VERSION = "1.6.2"
+  VERSION = "1.6.3"
 end

data/spec/builder_spec.rb

@@ -2,7 +2,82 @@
 
 require 'spec_helper'
 
+module Kernel
+  def capture(stream)
+    begin
+      stream = stream.to_s
+      eval "$#{stream} = StringIO.new"
+      yield
+      result = eval("$#{stream}").string
+    ensure
+      eval "$#{stream} = #{stream.upcase}"
+    end
+    result
+  end
+end
+
 describe Oddb2xml::Builder do
-  it 'pending'
+  include ServerMockHelper
+  before(:all) do
+    files = (Dir.glob('oddb_*.tar.gz')+ Dir.glob('oddb_*.zip')+ Dir.glob('oddb_*.xml'))# +Dir.glob('data/download/*'))
+    files.each{ |file| FileUtils.rm(file) }
+  end
+
+  before(:each) do
+    setup_server_mocks
+    setup_swiss_index_server_mock(types =  ['NonPharma', 'Pharma'])
+  end
+  context 'should handle BAG-articles with and without pharmacode' do
+    subject do
+      dat = File.read(File.expand_path('../data/Preparation.xml', __FILE__))
+      @items = Oddb2xml::BagXmlExtractor.new(dat).to_hash
+    end
+    it {
+      saved =  @items.clone
+      @items = subject.to_hash
+      expect(@items.size).to eq(2)
+      expect(saved).to eq(@items)
+    }
+  end if false
+  
+  context 'when no option is given' do
+    let(:cli) do
+      opts = {}
+      Oddb2xml::Cli.new(opts)
+    end    
+    it 'should generate a valid oddb_product.xml' do
+      res = capture(:stdout){ cli.run }
+      res.should match(/products/)
+      article_filename = File.expand_path(File.join(File.dirname(__FILE__), '..', 'oddb_article.xml'))
+      File.exists?(article_filename).should be_true   
+      article_xml = IO.read(article_filename)
+      product_filename = File.expand_path(File.join(File.dirname(__FILE__), '..', 'oddb_product.xml'))
+      File.exists?(product_filename).should be_true         
+      product_xml = IO.read(product_filename)
+      article_xml.should match(/3TC/) 
+      article_xml.should match(/<PHAR>1699947</)
+      article_xml.should match(/<SMNO>53662013</) 
+      article_xml.should match(/<DSCRD>3TC Filmtabl 150 mg</) 
+      article_xml.should match(/<COMPNO>7601001392175</) 
+      article_xml.should match(/<BC>7680536620137</) 
+      article_xml.should match(/<VDAT>01.10.2011</) 
+      article_xml.should match(/<PTYP>PEXF</) 
+      article_xml.should match(/<PRICE>164.55</) 
+      article_xml.should match(/<PTYP>PPUB</) 
+      article_xml.should match(/<PRICE>205.3</) 
+      
+      article_xml.should match(/Levetiracetam DESITIN/i) # 
+      article_xml.should match(/7680536620137/) # Pharmacode
+      article_xml.should match(/<PRICE>13.49</) 
+      article_xml.should match(/<PRICE>27.8</) 
+
+      product_xml.should match(/3TC/)
+      product_xml.should match(/7680620690084/) # Levetiracetam DESITIN
+      article_xml.scan(/<ART DT=/).size.should eq(2) # we should find two articles
+      article_xml.should match(/<PHAR>5819012</)
+      article_xml.should match(/<DSCRD>LEVETIRACETAM DESITIN Filmtabl 250 mg/)
+      article_xml.should match(/<COMPNO>7601001320451</)
+    end
+  end
 end
 

data/spec/cli_spec.rb

@@ -46,6 +46,10 @@ end
 
 describe Oddb2xml::Cli do
   include ServerMockHelper
+  before(:all) do
+    files = (Dir.glob('oddb_*.tar.gz')+ Dir.glob('oddb_*.zip')+ Dir.glob('oddb_*.xml'))# +Dir.glob('data/download/*'))
+    files.each{ |file| FileUtils.rm(file) }
+  end
   before(:each) do
     setup_server_mocks
   end
@@ -72,7 +76,7 @@ describe Oddb2xml::Cli do
     it 'should not create any xml file' do
       capture(:stdout) { cli.run }.should match(/Pharma/)
       Dir.glob('oddb_*.xml').each do |file|
-        File.exists?(file).should be_nil
+        File.exists?(file).should be_false
       end
     end
     after(:each) do
@@ -104,7 +108,7 @@ describe Oddb2xml::Cli do
     it 'should not create any xml file' do
       capture(:stdout) { cli.run }.should match(/Pharma/)
       Dir.glob('oddb_*.xml').each do |file|
-        File.exists?(file).should be_nil
+        File.exists?(file).should be_false
       end
     end
     after(:each) do

data/spec/data/swissindex_pharma.xml

@@ -1,55 +1,41 @@
 <?xml version="1.0" encoding="utf-8"?>
-<soap:Envelope
-  xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
   <soap:Body>
-    <PHARMA
+    <PHARMA xmlns="http://swissindex.e-mediat.net/SwissindexPharma_out_V101" CREATION_DATETIME="2013-12-10T10:09:26.8198427+01:00">
+      <!-- spec/data/swissindex_pharma.xml. Values for spec/builder_spec.rb -->
       CREATION_DATETIME="2012-10-18T06:31:19.4687619+02:00"
       xmlns="http://example.com/test">
-      <ITEM DT="2012-10-18T00:00:00">
-        <GTIN>0000000000000</GTIN>
-        <PHAR>00000</PHAR>
+      <ITEM DT="2013-06-22T00:00:00">
+        <GTIN>7680536620137</GTIN>
+        <PHAR>1699947</PHAR>
         <STATUS>A</STATUS>
-        <STDATE>2002-05-23T00:00:00</STDATE>
+        <STDATE>1996-03-20T00:00:00</STDATE>
         <LANG>DE</LANG>
-        <DSCR>Foo 1000 mg</DSCR>
-        <ADDSCR>Amp</ADDSCR>
-        <ATC>A00AA00</ATC>
+        <DSCR>3TC Filmtabl 150 mg</DSCR>
+        <ADDSCR>60 Stk</ADDSCR>
+        <ATC>J05AF05</ATC>
         <COMP>
-          <NAME>EXAMPLE.COM AG</NAME>
-          <GLN>0000000000000</GLN>
+          <NAME>ViiV Healthcare GmbH</NAME>
+          <GLN>7601001392175</GLN>
         </COMP>
       </ITEM>
-      <ITEM DT="2012-10-18T00:00:00">
-        <GTIN>0000000000001</GTIN>
-        <PHAR>00001</PHAR>
+      <ITEM DT="2013-12-06T00:00:00">
+        <GTIN>7680620690084</GTIN>
+        <PHAR>5819012</PHAR>
         <STATUS>A</STATUS>
-        <STDATE>2002-05-23T00:00:00</STDATE>
+        <STDATE>2013-10-26T00:00:00</STDATE>
         <LANG>DE</LANG>
-        <DSCR>Bar 10 Stk</DSCR>
-        <ADDSCR>10 Stk</ADDSCR>
-        <ATC>A02AA02</ATC>
+        <DSCR>LEVETIRACETAM DESITIN Filmtabl 250 mg</DSCR>
+        <ADDSCR>30 Stk</ADDSCR>
+        <ATC>N03AX14</ATC>
         <COMP>
-          <NAME>EXAMPLE.ORG AG</NAME>
-        </COMP>
-      </ITEM>
-      <ITEM DT="2012-10-18T00:00:00">
-        <GTIN>0000000000002</GTIN>
-        <PHAR>00002</PHAR>
-        <STATUS>A</STATUS>
-        <STDATE>2002-05-23T00:00:00</STDATE>
-        <LANG>DE</LANG>
-        <DSCR>Baz 10 Tablet</DSCR>
-        <ADDSCR>5 Stk</ADDSCR>
-        <ATC>A03AA03</ATC>
-        <COMP>
-          <NAME>EXAMPLE.AC AG</NAME>
+          <NAME>Desitin Pharma GmbH</NAME>
+          <GLN>7601001320451</GLN>
         </COMP>
       </ITEM>
       <RESULT>
         <OK_ERROR>OK</OK_ERROR>
-        <NBR_RECORD>3</NBR_RECORD>
+        <NBR_RECORD>2</NBR_RECORD>
       </RESULT>
     </PHARMA>
   </soap:Body>

data/spec/extractor_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require 'spec_helper'
+require "#{Dir.pwd}/lib/oddb2xml/downloader"
 
 describe Oddb2xml::Extractor do
   it "pending"
@@ -41,27 +42,49 @@ describe Oddb2xml::SwissIndexExtractor do
   it "pending"
 end
 
-describe Oddb2xml::SwissmedicExtractor do
+describe Oddb2xml::BMUpdateExtractor do
   it "pending"
 end
 
-describe Oddb2xml::MigelExtractor do
+describe Oddb2xml::LppvExtractor do
   it "pending"
 end
 
-describe Oddb2xml::SwissmedicInfoExtractor do
+describe Oddb2xml::SwissIndexExtractor do
   it "pending"
 end
 
-describe Oddb2xml::EphaExtractor do
+describe Oddb2xml::MigelExtractor do
   it "pending"
 end
 
-describe Oddb2xml::MedregbmExtractor do
-  it "pending"
+describe Oddb2xml::SwissmedicInfoExtractor do
+  context 'when transfer.dat is empty' do
+    subject { Oddb2xml::SwissmedicInfoExtractor.new("") }
+    it { expect(subject.to_hash).to be_empty }
+  end
+  context 'can parse swissmedic_packages.xls' do
+    it {
+        filename = File.join(File.dirname(__FILE__), 'data/swissmedic_packages.xls')
+        bin = IO.read(filename)
+        @packs = Oddb2xml::SwissmedicExtractor.new(bin, :package).to_hash
+        expect(@packs.size).to eq(14) 
+       }
+  end
 end
 
-describe Oddb2xml::ZurroseExtractor do
+describe Oddb2xml::EphaExtractor do
+  context 'can parse epha_interactions.csv' do
+    it {
+        filename = File.join(File.dirname(__FILE__), 'data/epha_interactions.csv')
+        string = IO.read(filename)
+        @actions = Oddb2xml::EphaExtractor.new(string).to_arry
+        expect(@actions.size).to eq(2) 
+       }
+  end
+end
+
+describe Oddb2xml::MedregbmExtractor do
   it "pending"
 end
 

data/spec/spec_helper.rb

@@ -42,8 +42,8 @@ module ServerMockHelper
         :headers => {'Content-Type' => 'application/zip; charset=utf-8'},
         :body    => stub_response)
   end
-  def setup_swiss_index_server_mock
-    ['Pharma', 'NonPharma'].each do |type|
+  def setup_swiss_index_server_mock(types =  ['Pharma', 'NonPharma'])
+    types.each do |type|
       # wsdl
       stub_wsdl_url = "https://index.ws.e-mediat.net/Swissindex/#{type}/ws_#{type}_V101.asmx?WSDL"
       stub_response = File.read(File.expand_path("../data/wsdl_#{type.downcase}.xml", __FILE__))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oddb2xml
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.6.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-09 00:00:00.000000000 Z
+date: 2013-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip