oddb2xml 2.6.6 → 2.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: caafd3b48a18d05d29249b82717da28ca795d2c5003c69ce06ee10a7c7d6ecee
-  data.tar.gz: 7429e7b06a997e04aa5be4ab0883a9dbc4c6638a5c7421899d115c5196b8f853
+  metadata.gz: c31777bcd85038bd65d38d67d6ebdf2197f174389403b76ff146b18d3ef4557e
+  data.tar.gz: 8d8930ed04518838084ae4aeb69181418b2eddb36cfa83ece2cbc84ccd06e1f4
 SHA512:
-  metadata.gz: 5ec16fedb1d2185ae3a44c1f1e1ac1a333f162bb27cfcd3a46f4efb34a3ca8fb87de9a9e650aaabfa992145015db8c7ad4a8498597bbdf7d8ca528c595fb4b5a
-  data.tar.gz: 96bb2e0f902808245e8e5b58d012e785790feff065608bfada73ddaaf35a27e23ba4c3cbc3ba3afbee09f77c1bd53e99e3b691ea9d8f75c34a1baa01d896d0f8
+  metadata.gz: abe9685cc40ba1fbf9525c3cbeb9f332a98725e76621074c9f1df179c91e53cb50a4f15e78a800a28224351d3a47ec8aef0e3b9ee17787a8dccd519db65bc6d8
+  data.tar.gz: 415b9049a316af98279f7af0b602ecd85b0bd2e86493133af512d37b494d43155822ef754ad6dcaa39b1aa474fddba57081860588b7e896da13bf891e9b522d6

data/.github/workflows/ruby.yml

@@ -0,0 +1,40 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Ruby
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+# Using a matrix fails, because /setup-ruby always invokes bundle install without any additional args
+# Fixed by adding not defining the debugger group in the Gemfile
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu]
+        ruby: [2.4, 2.5, 2.6, 2.7, 3.0, head]
+    continue-on-error: ${{ endsWith(matrix.ruby, 'head') }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+
+    - name: Install dependencies
+      run: |
+        bundle config set path .bundle/gems --local
+        bundle config set without debugger  --local
+        bundle install
+    - name: Test with RSpec
+      run: bundle exec rspec

data/Elexis_Artikelstamm_v5.xsd

@@ -1,11 +1,11 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!-- Copyright (c) 2016 MEDEVIT. All rights reserved. This program and the 
-	accompanying materials are made available under the terms of the Eclipse 
-	Public License v1.0 which accompanies this distribution, and is available 
-	at http://www.eclipse.org/legal/epl-v10.html v002 - Ergänzung K Komplementärprodukt 
-	bei generic type, Consider ATC code of length 8 (on vet.med. articles) Relax 
-	ADDSCR occurence to 0 .. 1 v003 - PRODNO hinzugefügt LIMNAMEBAG zur formal-kennzeichnung 
-	von identen limitations v4 - Major refactoring, see https://redmine.medelexis.ch/issues/4405 
+<!-- Copyright (c) 2016 MEDEVIT. All rights reserved. This program and the
+	accompanying materials are made available under the terms of the Eclipse
+	Public License v1.0 which accompanies this distribution, and is available
+	at http://www.eclipse.org/legal/epl-v10.html v002 - Ergänzung K Komplementärprodukt
+	bei generic type, Consider ATC code of length 8 (on vet.med. articles) Relax
+	ADDSCR occurence to 0 .. 1 v003 - PRODNO hinzugefügt LIMNAMEBAG zur formal-kennzeichnung
+	von identen limitations v4 - Major refactoring, see https://redmine.medelexis.ch/issues/4405
 	Contributors: MEDEVIT <office@medevit.at> - initial API and implementation -->
 <xs:schema targetNamespace="http://elexis.ch/Elexis_Artikelstamm_v5"
 	elementFormDefault="qualified" attributeFormDefault="unqualified"
@@ -339,7 +339,6 @@
 										</xs:element>
 										<xs:element name="GENERIC_TYPE"
 											maxOccurs="1" minOccurs="0">
-
 											<xs:simpleType>
 												<xs:annotation>
 													<xs:documentation>
@@ -359,6 +358,9 @@
 													<xs:enumeration
 														value="K">
 													</xs:enumeration>
+													<xs:enumeration
+                            value="C"> <!-- BAG 9.6.2020: BDer Buchstabe C bei «Original Generika Code» steht für Co-Marketing Arzneimittel -->
+													</xs:enumeration>
 												</xs:restriction>
 											</xs:simpleType>
 										</xs:element>

data/History.txt

@@ -1,3 +1,30 @@
+=== 2.7.1 / 03.02.2021
+* travis-ci -> Github Actions
+* limit DSCRD/DSCRF to 120 chars as specified in the XSD
+* Use huge option for nokogiri as suggested by  https://github.com/sparklemotion/nokogiri/issues/2187
+* Adapt artikelstamm to refdata without pharmacode
+* oddb_article get pharmacode where possible from transfer.dat
+
+=== 2.7.0 / 05.11.2020
+* Chapter70 hack nur als Kommentar
+* Avoid double hyphen in comment
+
+=== 2.6.9 / 29.06.2020
+
+* Fix problem when running bin/oddb2xml -a -f dat. Thanks to Lukas Furre for reporting the problem
+* Fix warning with regular expression thas has ']'
+
+=== 2.6.8 / 26.06.2020
+
+* Re-Add support for Ruby Version 2.4
+* Specified minimal ruby version in gemspec
+* Patch artikelstamm build to emit a product for Varilrix
+
+=== 2.6.7 / 25.06.2020
+* Added value 'C' (Co-marketing) for field GENERIC_TYPE Elexis_Artikelstamm_v5.xsd from BAG Preparations.xml
+* Fixed limitations coming from the chapte 70 hack
+* Ruby 2.7 compatible (No URI.open warning)
+
 === 2.6.6 / 11.01.2020
 * Fix link for varia_De.htm
 
@@ -130,7 +157,7 @@
 
 === 2.4.1 / 2.12.2016
 
-* Update oddb2xml.gemspec to add GPL-3.0 as correct license for Rubygems as stated in LICENSE file 
+* Update oddb2xml.gemspec to add GPL-3.0 as correct license for Rubygems as stated in LICENSE file
 * Added a hack for 24 globulina equina which commas between brackets
 
 === 2.4.0 /25.08.2016
@@ -568,7 +595,7 @@
 
 * Improve NINCD detection
   - Update timeout handling
-  - Improve pharmacode detection                
+  - Improve pharmacode detection
 
 === 1.5.1 / 10.06.2013
 

data/bin/check_artikelstamm

@@ -16,7 +16,7 @@ require 'optimist'
   #{File.basename(__FILE__)} xml_file_to_check
 EOS
 end
-$stdout.sync = false
+$stdout.sync = true
 
 args = ARGV
 if args.size != 1

data/lib/oddb2xml/builder.rb

@@ -520,7 +520,7 @@ module Oddb2xml
           return false
         end
         return false if !name || name.empty? || name.length < 3
-        name
+        name[0..119] # limit to maximal 120 chars as specified in the XSD
       end
       prepare_substances
       prepare_products
@@ -886,8 +886,12 @@ module Oddb2xml
             xml.ART('DT' => obj[:last_change] ? obj[:last_change] : '') do
               nbr_records += 1
               xml.REF_DATA (obj[:refdata] || @migel[pharma_code]) ? '1' : '0'
-              # xml.PHAR  sprintf('%07d', obj[:pharmacode]) if obj[:pharmacode]
-              xml.PHAR  obj[:pharmacode] if obj[:pharmacode]
+              if obj[:pharmacode] && obj[:pharmacode].length > 0
+                xml.PHAR  obj[:pharmacode]
+              elsif zur_rose
+                puts "Adding #{zur_rose[:pharmacode]} to article GTIN #{ean}"
+                xml.PHAR  zur_rose[:pharmacode]
+              end
               #xml.GRPCD
               #xml.CDS01
               #xml.CDS02
@@ -988,14 +992,7 @@ module Oddb2xml
                 xml.CDTYP  'E13'
                 xml.BC     ean #  /^9999|^0000|^0$/.match(ean.to_s) ? 0 : sprintf('%013d', ean)
                 xml.BCSTAT 'A' # P is alternative
-                #xml.PHAR2
               } if ean
-              #xml.ARTCH {
-                #xml.PHAR2
-                #xml.CHTYPE
-                #xml.LINENO
-                #xml.NOUNITS
-              #}
               if pac and pac[:prices]
                 pac[:prices].each_pair do |key, price|
                   xml.ARTPRI {
@@ -1337,7 +1334,7 @@ module Oddb2xml
                                           else
                                             ('0' * DAT_LEN[:ITHE])
                                           end.to_i
-          row << "%0#{DAT_LEN[:CEAN]}d" % (sprintf('%013d', ean).match(/^000000/) ? 0 : ean.to_i)
+          row << "%0#{DAT_LEN[:CEAN]}d" % (sprintf('%013d', ean.to_i).match(/^000000/) ? 0 : ean.to_i)
           row << "%#{DAT_LEN[:CMWS]}s"  % '2' # pharma
           rows << row
       end
@@ -1413,7 +1410,7 @@ module Oddb2xml
           return '--missing--'
         end
         return '--missing--' if !name || name.empty? || name.length < 3
-        name
+        name[0..119] # limit to maximal 120 chars as specified in the XSD
       end
       def override(xml, id, field, default_value)
         has_overrides =  /\d{13}/.match(id.to_s) ? @@article_overrides[id.to_i] : @@product_overrides[id.to_i]
@@ -1499,7 +1496,6 @@ module Oddb2xml
                 name ||= (item[:desc_de] + item[:name_de]) if item
                 name ||= obj[:sequence_name]
                 xml.GTIN pkg_gtin.to_s.rjust(13, '0')
-                override(xml, pkg_gtin, :PHAR, pharma_code)
                 xml.SALECD('A')
                 # maxLength for DSCR is 50 for Artikelstamm v3
                 xml.DSCR(name) # for description for zur_rose
@@ -1557,9 +1553,20 @@ module Oddb2xml
                   when 'N'; xml.DEDUCTIBLE 10; # 10%
                 end if item && item[:deductible]
                 prodno = Oddb2xml.getProdnoForEan13(pkg_gtin)
-                xml.PRODNO prodno if prodno
                 atc = package[:atc_code]
                 atc ||= @refdata[pkg_gtin][:seq][:atc_code] if @refdata[pkg_gtin]
+                unless prodno # find a prodno from packages for vaccinations
+                  if atc && /^J07/.match(atc) && !/^J07AX/.match(atc)
+                    pack = @packs.values.find{ |v| v && v[:atc_code].eql?(atc)}
+                    if pack
+                      prodno = pack[:prodno]
+                      Oddb2xml.log "Patching vaccination for #{pkg_gtin} #{atc} #{name} via prodno #{prodno}"
+                    else
+                      Oddb2xml.log "unable to find a pack/prodno for  vaccination for #{pkg_gtin} #{atc} #{name}"
+                    end
+                  end
+                end
+                xml.PRODNO prodno if prodno
                 csv = []
                 @csv_file << [pkg_gtin, name, package[:unit], measure,
                               pexf ? pexf : '',
@@ -1571,8 +1578,6 @@ module Oddb2xml
               end
             end
           else # non pharma
-             @csv_file << [ ean13, (obj[:desc_de] || obj[:description]), '', '',
-                          obj[:price], obj[:pub_price], '', '', '', '', '', '' ]
             if @@emitted_v5_gtins.index(ean13)
               next
             else
@@ -1589,17 +1594,37 @@ module Oddb2xml
             next if /^#{Oddb2xml::FAKE_GTIN_START}/.match(ean13.to_s)
             xml.ITEM({'PHARMATYPE' => patched_pharma_type }) do
               xml.GTIN ean13.to_s.rjust(13, '0')
-              xml.PHAR obj[:pharmacode]
+              xml.PHAR obj[:pharmacode] if obj[:pharmacode] && obj[:pharmacode].length > 0
               emit_salecd(xml, ean13, obj)
-              xml.DSCR(obj[:desc_de] || obj[:description]) # for description for zur_rose
+              description = obj[:desc_de] || obj[:description] # for description for zur_rose
+              xml.DSCR(description)
               xml.DSCRF(obj[:desc_fr] || '--missing--')
               xml.COMP  do
                 xml.GLN obj[:company_ean]
               end if obj[:company_ean] && !obj[:company_ean].empty?
-              xml.PEXF obj[:price]      if obj[:price] && !obj[:price].empty?
-              xml.PPUB obj[:pub_price]  if obj[:pub_price] && !obj[:pub_price].empty?
+              if !(obj[:price] && !obj[:price].empty?) || !(obj[:pub_price] && !obj[:pub_price].empty?)
+                zur_rose_detail = @infos_zur_rose.values.find{|x| x[:ean13].to_i == ean13.to_i}
+              end
+              ppub = nil; pexf=nil
+              if obj[:price] && !obj[:price].empty?
+                xml.PEXF (pexf = obj[:price])
+              elsif zur_rose_detail
+                if zur_rose_detail[:price] && !zur_rose_detail[:price].empty? && !zur_rose_detail[:price].eql?('0.00')
+                  Oddb2xml.log "NonPharma: #{ean13} adding PEXF #{zur_rose_detail[:price]} #{description}"
+                  xml.PEXF (pexf = zur_rose_detail[:price])
+                end
+              end
+              if obj[:pub_price] && !obj[:pub_price].empty?
+                xml.PPUB  (ppub = obj[:pub_price])
+              elsif zur_rose_detail
+                if zur_rose_detail[:pub_price] && !zur_rose_detail[:pub_price].empty? && !zur_rose_detail[:pub_price].eql?('0.00')
+                  Oddb2xml.log "NonPharma: #{ean13} adding PPUB #{zur_rose_detail[:pub_price]} #{description}"
+                  xml.PPUB (ppub = zur_rose_detail[:pub_price])
+                end
+              end
+             @csv_file << [ ean13, description, '', '', pexf, ppub, '', '', '', '', '', '' ]
               if chap70
-                xml.comment "Chapter70 hack" 
+                xml.comment "Chapter70 hack #{ean13.to_s.rjust(13, '0')} #{description.encode(:xml => :text).gsub('--','-')}"
                 xml.SL_ENTRY 'true'
                 xml.PRODNO obj[:pharmacode]
               end
@@ -1665,7 +1690,7 @@ module Oddb2xml
               myPack = @packs.values.find{ |x| x[:iksnr].to_i == obj[:seq][:swissmedic_number5].to_i } if  obj[:seq]
               if myPack && !prodno
                 prodno ||= myPack[:prodno]
-                puts "Setting prodno #{prodno} for #{ean13} Varilrix"
+                puts "Setting prodno #{prodno} for #{ean13} #{myPack[:sequence_name]}"
               end
               next unless prodno
               next if emitted_prodno.index(prodno)
@@ -1703,12 +1728,14 @@ module Oddb2xml
                   used_limitations << lim_code unless used_limitations.index(lim_code)
                   xml.LIMNAMEBAG lim_code
                 elsif obj[:chapter70]
-                  used_limitations << obj[:code]
-                  xml.comment "Chapter70 hack"
+                  xml.comment "Chapter70 hack prodno #{prodno} #{obj[:description].encode(:xml => :text).gsub('--','-')}"
                   xml.SALECD('A') # these products are always active!
                   xml.DSCR obj[:description]
                   xml.DSCRF ''
-                  xml.LIMNAMEBAG obj[:code]
+                  if @limitations.index(obj[:code])
+                    xml.LIMNAMEBAG obj[:code]
+                    used_limitations << obj[:code]
+                  end
                 end
                 if sequence && sequence[:substances]
                   value = nil
@@ -1734,7 +1761,7 @@ module Oddb2xml
               end
               emitted_lim_code << lim[:code]
               xml.LIMITATION do
-                xml.comment "Chapter70 hack" if lim[:chap70]
+                xml.comment "Chapter70 2 hack" if lim[:chap70]
                 xml.LIMNAMEBAG lim[:code] # original LIMCD
                 xml.DSCR       Oddb2xml.html_decode(lim[:desc_de])
                 xml.DSCRF      Oddb2xml.html_decode(lim[:desc_fr])

data/lib/oddb2xml/chapter_70_hack.rb

@@ -12,7 +12,7 @@ module Oddb2xml
         puts "parse_td returns: #{res}" if $VERBOSE
       rescue => exc
         puts "Unable to pars #{elem} #{exc}"
-        binding.pry
+        # binding.pry
         return nil
       end
       res.flatten # .join("\t")
@@ -33,7 +33,7 @@ module Oddb2xml
           effort: :tolerant,
           smart:  true
       }
-      res = Ox.load(open(html_file).read, mode: :hash_no_attrs).values.first['body']
+      res = Ox.load(Oddb2xml.uri_open(html_file).read, mode: :hash_no_attrs).values.first['body']
       result = []
       idx = 0
       @@items = {}
@@ -50,15 +50,16 @@ module Oddb2xml
         data = {}
         pharma_code = entry.first
         ean13 =  (Oddb2xml::FAKE_GTIN_START + pharma_code.to_s)
-        german = entry[2].force_encoding('ISO-8859-1').encode('UTF-8')
-        while !german.eql?(HTMLEntities.new.decode(german))
-          german = HTMLEntities.new.decode(german)
+        if entry[2].encoding.to_s.eql?('ASCII-8BIT')
+          german = CGI.unescape(entry[2].force_encoding('ISO-8859-1'))
+        else
+          german = entry[2]
         end
         @@items[ean13] = {
           :data_origin   => 'Chapter70',
           :line   => entry.join(","),
           :ean13 => ean13,
-          :description => Oddb2xml.html_decode(entry[2]),
+          :description => german,
           :quantity => entry[3],
           :pharmacode => pharma_code,
           :pub_price => entry[4],
@@ -70,5 +71,3 @@ module Oddb2xml
     end
   end
 end
-
-  

data/lib/oddb2xml/cli.rb

@@ -88,6 +88,7 @@ module Oddb2xml
             puts "Validatied #{@_files[:artikelstamm]}"
          else
             puts "Validating failed using #{cmd}"
+            raise  "Validating failed using #{cmd}"
             exit(2)
          end
       end
@@ -157,7 +158,9 @@ module Oddb2xml
             end
           end
           if @options[:calc]
-            FileUtils.cp(File.join(WorkDir, file), File.join(WorkDir, file.sub('.xml', '_'+Time.now.strftime("%d.%m.%Y_%H.%M")+'.xml')), :verbose => false)
+            ext = File.extname(file)
+            dest = File.join(WorkDir, file.sub(ext, '_'+Time.now.strftime("%d.%m.%Y_%H.%M")+ext))
+            FileUtils.cp(File.join(WorkDir, file), dest, :verbose => false)
           end
         end
       rescue Interrupt

data/lib/oddb2xml/downloader.rb

@@ -23,7 +23,7 @@ module Oddb2xml
       else
         begin
           io = File.open(file, option)
-          data = open(@url).read
+          data = Oddb2xml.uri_open(@url).read
           io.write(data)
         rescue => error
           puts "error #{error} while fetching #{@url}"
@@ -283,7 +283,7 @@ module Oddb2xml
     include DownloadMethod
     def initialize(type=:orphan, options = {})
       url = BASE_URL + '/swissmedic/de/home/services/listen_neu.html'
-      doc = Nokogiri::HTML(open(url))
+      doc = Nokogiri::HTML(Oddb2xml.uri_open(url))
       @type = type
       @options = options
       case @type

data/lib/oddb2xml/extractor.rb

@@ -218,7 +218,6 @@ module Oddb2xml
         item = {}
         item[:ean13]           = ean13
         item[:no8]             = pac.SWMC_AUTHNR
-        item[:pharmacode]      = (phar = pac.PHAR.to_s)   ? phar: '0'
         item[:data_origin]     = 'refdata'
         item[:refdata]         = true
         item[:_type]           = (typ  = pac.ATYPE.downcase.to_sym)  ? typ: ''
@@ -228,12 +227,6 @@ module Oddb2xml
         item[:atc_code]        = (code = pac.ATC)    ? code.to_s : ''
         item[:company_name] = (nam = pac.AUTH_HOLDER_NAME) ? nam: ''
         item[:company_ean]  = (gln = pac.AUTH_HOLDER_GLN)  ? gln: ''
-        unless item[:pharmacode]
-          item[:pharmacode] = phar
-          unless data[item[:pharmacode]] # pharmacode => GTINs
-            data[item[:ean13]] = []
-          end
-        end
         data[item[:ean13]] = item
       end
       data

data/lib/oddb2xml/parslet_compositions.rb

@@ -88,7 +88,11 @@ module ParseUtil
         composition.substances.each do
           |substance_item|
           active_substance_name = substance_item.name.downcase.sub(/^cum\s/, '')
-          substance_item.is_active_agent = (active_agents.find {|x| /#{x.downcase.gsub('(', '\(').gsub(')', '\)')}($|\s)/.match(active_substance_name) } != nil)
+          substance_item.is_active_agent = (active_agents.find {|x| /#{x.downcase.
+                                                                gsub('(', '\(').
+                                                                gsub(')', '\)').
+                                                                gsub('[', '\[').
+                                                                gsub(']', '\]')}($|\s)/.match(active_substance_name) } != nil)
           substance_item.is_active_agent = true if substance_item.chemical_substance and active_agents.find {|x| x.downcase.eql?(substance_item.chemical_substance.name.downcase) }
          end
         comps << composition

data/lib/oddb2xml/semantic_check.rb

@@ -64,7 +64,7 @@ module Oddb2xml
         end
       end
     end
-    
+
     def everyPharmaArticleHasAProductItem
       result = true
       puts "#{Time.now.strftime("%H:%M:%S")}: everyPharmaArticleHasAProductItem"
@@ -74,7 +74,7 @@ module Oddb2xml
         unless allProductNumbers.index(item[:PRODNO])
           puts "Item #{item[:GTIN]}  has no Product #{item[:PRODNO]}  #{item[:DSCR]}"
           result = false
-        end
+        end unless item[:Chapter70_HACK]
       end
       result
     end
@@ -91,17 +91,19 @@ module Oddb2xml
       end
       result
     end
-    
+
     def everyReferencedLimitationIsIncluded
+      result = true
       puts "#{Time.now.strftime("%H:%M:%S")}: everyReferencedLimitationIsIncluded"
       allLimitations = limitations.collect{ |lim| lim[:LIMNAMEBAG] }
       products.each do |product|
         next unless product[:LIMNAMEBAG]
         unless allLimitations.index(product[:LIMNAMEBAG])
           puts "product #{product[:PRODNO]}  has no limitation #{product[:LIMNAMEBAG]} #{product[:DSCR]}"
-          return false
+          result = false
         end
       end
+      result
     end
 
     def checkPackageSize