oddb2xml 1.9.4 → 1.9.5

data/data/gal_groups.yaml

@@ -0,0 +1,217 @@
+---
+754720: !ruby/struct:Struct::GalenicGroup
+  oid: 754720
+  descriptions:
+    de: Kaugummi
+    en: Chewing Gum
+    fr: Chewing Gum
+890150: !ruby/struct:Struct::GalenicGroup
+  oid: 890150
+  descriptions:
+    de: Brausetabletten
+    en: Solutabs
+    fr: Comprimés Effervescents
+754487: !ruby/struct:Struct::GalenicGroup
+  oid: 754487
+  descriptions:
+    de: Dialyse
+    en: Dialysis
+    fr: Dialyse
+890432: !ruby/struct:Struct::GalenicGroup
+  oid: 890432
+  descriptions:
+    de: Tropfen
+    en: Drops
+    fr: Gouttes
+7830083: !ruby/struct:Struct::GalenicGroup
+  oid: 7830083
+  descriptions:
+    de: Lösbar zur Einnahme Retard
+    en: Delayed releas soluble for ingestion
+    fr: Soluble pour ingestion retard
+1470796: !ruby/struct:Struct::GalenicGroup
+  oid: 1470796
+  descriptions:
+    de: Klistiere
+    en: Enemas
+    fr: Lavements
+714576: !ruby/struct:Struct::GalenicGroup
+  oid: 714576
+  descriptions:
+    de: Klebstoff
+    en: Adhesive
+    fr: Gluten
+1479254: !ruby/struct:Struct::GalenicGroup
+  oid: 1479254
+  descriptions:
+    de: Tests
+    en: Tests
+    fr: Tests
+9672801: !ruby/struct:Struct::GalenicGroup
+  oid: 9672801
+  descriptions:
+    de: Lösungsmittel
+    en: Solvent
+    fr: Solvant
+1470567: !ruby/struct:Struct::GalenicGroup
+  oid: 1470567
+  descriptions:
+    de: Subkutane Implantate
+    en: Hypodermic Implants
+    fr: Implants hypodermiques
+890480: !ruby/struct:Struct::GalenicGroup
+  oid: 890480
+  descriptions:
+    de: Gastrointenstinales Therapiesystem
+    en: Gastro-Intestinal Therapeutic System
+    fr: Système Therapeutique Gastro-Intestinale
+9672817: !ruby/struct:Struct::GalenicGroup
+  oid: 9672817
+  descriptions:
+    de: Tupfer/Gaze
+    en: Swab/Gauze
+    fr: Compresse/Gaze
+1478018: !ruby/struct:Struct::GalenicGroup
+  oid: 1478018
+  descriptions:
+    de: Tinkturen/Desinfektion
+    en: Tinctures/Disinfection
+    fr: Teintures/Désinfection
+1471897: !ruby/struct:Struct::GalenicGroup
+  oid: 1471897
+  descriptions:
+    de: Lösbar zur Injektion/Infusion
+    en: Soluble for injection/infusion
+    fr: Soluble pour injection/infusion
+1472921: !ruby/struct:Struct::GalenicGroup
+  oid: 1472921
+  descriptions:
+    de: Rektal-Produkte
+    en: Rectal Products
+    fr: Produits Rectaux
+1471899: !ruby/struct:Struct::GalenicGroup
+  oid: 1471899
+  descriptions:
+    de: Seifen und Shampoos
+    en: Soaps and Shampoos
+    fr: Savons et Shampoos
+1472699: !ruby/struct:Struct::GalenicGroup
+  oid: 1472699
+  descriptions:
+    de: Buccal-/Dentalprodukte
+    en: Buccal/dental products
+    fr: Produits buccodentaires
+1472713: !ruby/struct:Struct::GalenicGroup
+  oid: 1472713
+  descriptions:
+    de: Spray/Puder
+    en: Spray/Powder
+    fr: Spray/Poudre
+1472742: !ruby/struct:Struct::GalenicGroup
+  oid: 1472742
+  descriptions:
+    de: Trinkflüssigkeiten Retard
+    en: Delayed release drinkable liquids
+    fr: Liquides buvables retard
+1: !ruby/struct:Struct::GalenicGroup
+  oid: 1
+  descriptions:
+    de: unbekannt
+    en: unbekannt
+    fr: unbekannt
+2: !ruby/struct:Struct::GalenicGroup
+  oid: 2
+  descriptions:
+    de: Tabletten
+    en: Tablets
+    fr: Comprimés
+3: !ruby/struct:Struct::GalenicGroup
+  oid: 3
+  descriptions:
+    de: Injektion/Infusion
+    en: Injection/Infusion
+    fr: Injection/Infusion
+4: !ruby/struct:Struct::GalenicGroup
+  oid: 4
+  descriptions:
+    de: Salben
+    en: Ointments
+    fr: Onguents
+5: !ruby/struct:Struct::GalenicGroup
+  oid: 5
+  descriptions:
+    de: Augenmittel
+    en: Ophtalmics
+    fr: Préparations oculaires
+6: !ruby/struct:Struct::GalenicGroup
+  oid: 6
+  descriptions:
+    de: Inhalation
+    en: Inhalation
+    fr: Inhalation
+7: !ruby/struct:Struct::GalenicGroup
+  oid: 7
+  descriptions:
+    de: Lösbar zur Einnahme
+    en: Soluble for ingestion
+    fr: Soluble pour ingestion
+8: !ruby/struct:Struct::GalenicGroup
+  oid: 8
+  descriptions:
+    de: Badezusatz
+    en: Bath additives
+    fr: Bains
+9: !ruby/struct:Struct::GalenicGroup
+  oid: 9
+  descriptions:
+    de: Essbare
+    en: Edibles
+    fr: Denrées
+10: !ruby/struct:Struct::GalenicGroup
+  oid: 10
+  descriptions:
+    de: Lutschtabletten
+    en: Lozenges
+    fr: Comprimés à succer
+11: !ruby/struct:Struct::GalenicGroup
+  oid: 11
+  descriptions:
+    de: Trinkflüssigkeiten
+    en: Drinkable liquids
+    fr: Liquides buvables
+12: !ruby/struct:Struct::GalenicGroup
+  oid: 12
+  descriptions:
+    de: Zubehör
+    en: Other
+    fr: Autres
+13: !ruby/struct:Struct::GalenicGroup
+  oid: 13
+  descriptions:
+    de: Nasenmittel
+    en: Nasal products
+    fr: Remèdes nasales
+14: !ruby/struct:Struct::GalenicGroup
+  oid: 14
+  descriptions:
+    de: Retard-Tabletten
+    en: Delayed release tablets
+    fr: Comprimés retards
+15: !ruby/struct:Struct::GalenicGroup
+  oid: 15
+  descriptions:
+    de: Vaginal-Produkte
+    en: Vaginal products
+    fr: Produits Vaginaux
+16: !ruby/struct:Struct::GalenicGroup
+  oid: 16
+  descriptions:
+    de: Pflaster/Transdermale Systeme
+    en: Patches/Transdermal Systems
+    fr: Pansements/Systèmes transdermales
+17: !ruby/struct:Struct::GalenicGroup
+  oid: 17
+  descriptions:
+    de: Suppositorien
+    en: Suppositories
+    fr: Suppositoires

data/lib/oddb2xml/builder.rb

@@ -1,6 +1,10 @@
 # encoding: utf-8
 
 require 'nokogiri'
+require 'oddb2xml/util'
+require 'oddb2xml/calc'
+require 'csv'
+
 class Numeric
   # round a given number to the nearest step
   def round_by(increment)
@@ -60,7 +64,7 @@ module Oddb2xml
       end
     end
     def to_xml(subject=nil)
-      Oddb2xml.log "to_xml subject #{subject ? subject.to_s :  @subject.to_s} for #{self.class}"
+      Oddb2xml.log "to_xml subject #{subject} #{@subject} for #{self.class}"
       if subject
         self.send('build_' + subject.to_s)
       elsif @subject
@@ -232,6 +236,7 @@ module Oddb2xml
         @products = []
         @index['DE'].each_pair do |phar, indices|
           indices.each_with_index do |index, i|
+            next if index and index.is_a?(Hash) and index[:atc_code] and /^Q/i.match(index[:atc_code])
             obj = {
               :seq => @items[phar] ? @items[phar] : @items[index[:ean]],
               :pac => nil,
@@ -246,6 +251,7 @@ module Oddb2xml
               :siz => '',
               :eht => '',
               :sub => '',
+              :comp => '',
             }
             if obj[:ean] # via EAN-Code
               obj[:no8] = obj[:ean][4..11]
@@ -264,6 +270,7 @@ module Oddb2xml
               obj[:siz] = ppac[:package_size]
               obj[:eht] = ppac[:einheit_swissmedic]
               obj[:sub] = ppac[:substance_swissmedic]
+              obj[:comp] = ppac[:composition_swissmedic]
             end
             if obj[:ean][0..3] == '7680'
               @products << obj
@@ -443,6 +450,7 @@ module Oddb2xml
           list = []
           length = 0
           @products.each do |obj|
+            next if /^Q/i.match(obj[:atc])
             seq = obj[:seq]
             length += 1
               xml.PRD('DT' => '') {
@@ -567,6 +575,7 @@ module Oddb2xml
               xml.PackGrSwissmedic    obj[:siz] unless obj[:siz].empty?
               xml.EinheitSwissmedic   obj[:eht] unless obj[:eht].empty?
               xml.SubstanceSwissmedic obj[:sub] unless obj[:sub].empty?
+              xml.CompositionSwissmedic obj[:comp] unless obj[:comp].empty?
             }
           end
           xml.RESULT {
@@ -579,6 +588,69 @@ module Oddb2xml
       end
       _builder.to_xml
     end
+
+    def build_calc
+      packungen_xlsx = File.join(Oddb2xml::WorkDir, "swissmedic_package.xlsx")
+      idx = 0
+      workbook = RubyXL::Parser.parse(packungen_xlsx)
+      items = {}
+      row_nr = 0
+      _builder = Nokogiri::XML::Builder.new(:encoding => 'utf-8') do |xml|
+        xml.doc.tag_suffix = @tag_suffix
+        datetime = Time.new.strftime('%FT%T%z')
+        xml.ARTICLES(XML_OPTIONS) {
+          workbook.worksheets[0].each do |row|
+            row_nr += 1
+            next unless row and row.cells[0] and row.cells[0].value and row.cells[0].value.to_i > 0
+            iksnr               = "%05i" % row.cells[0].value.to_i
+            seqnr               = "%02d" % row.cells[1].value.to_i
+            no8                 = sprintf('%05d',row.cells[0].value.to_i) + sprintf('%03d',row.cells[10].value.to_i)
+            name                = row.cells[2].value
+            atc_code            = row.cells[5]  ? row.cells[5].value  : nil
+            list_code           = row.cells[6]  ? row.cells[6].value  : nil
+            package_size        = row.cells[11] ? row.cells[11].value : nil
+            unit                = row.cells[12] ? row.cells[12].value : nil
+            active_substance    = row.cells[14] ? row.cells[14].value : nil
+            composition         = row.cells[15] ? row.cells[15].value : nil
+
+            # skip veterinary product
+            next if atc_code  and /^Q/i.match(atc_code)
+            next if list_code and /Tierarzneimittel/.match(list_code)
+
+            info = Calc.new(name, package_size, unit, composition)
+            ean12 = '7680' + no8
+            ean13 = (ean12 + Oddb2xml.calc_checksum(ean12))
+            items[ean13] = info
+            xml.ARTICLE {
+              xml.GTIN          ean13
+              xml.NAME          name
+              xml.PKG_SIZE      package_size
+              xml.SELLING_UNITS info.selling_units
+#              xml.COUNT         info.count
+#              xml.MULTI         info.multi
+              xml.MEASURE       info.measure # Nur wenn Lösung wen Spalte M ml, Spritze
+#              xml.ADDITION      info.addition
+#              xml.SCALE         info.scale
+              if  info.galenic_form.is_a?(String)
+                xml.GALENIC_FORM  info.galenic_form
+                xml.GALENIC_GROUP "Unknown"
+              else
+                xml.GALENIC_FORM  info.galenic_form.description
+                xml.GALENIC_GROUP info.galenic_group ? info.galenic_group.description : "Unknown"
+              end
+            }
+          end
+        }
+      end
+      csv_name = File.join(WorkDir, 'oddb_calc.csv')
+      CSV.open(csv_name, "w+", :col_sep => ';') do |csv|
+        csv << ['gtin'] + items.values.first.headers
+        items.each do |key, value|
+          csv <<  [key] + value.to_array
+        end
+      end
+      _builder.to_xml
+    end
     def build_article
       prepare_limitations
       prepare_articles

data/lib/oddb2xml/calc.rb

@@ -0,0 +1,327 @@
+# encoding: utf-8
+
+require 'oddb2xml/util'
+require 'yaml'
+
+module Oddb2xml
+ # Calc is responsible for analysing the columns "Packungsgrösse" and "Einheit"
+ #
+  GalenicGroup  = Struct.new("GalenicGroup", :oid, :descriptions)
+  GalenicForm   = Struct.new("GalenicForm",  :oid, :descriptions, :galenic_group)
+
+  class GalenicGroup
+    def description(lang = 'de')
+      descriptions[lang]
+    end
+  end
+
+  class GalenicForm
+    def description(lang = 'de')
+      descriptions[lang]
+    end
+  end
+
+  class Calc
+    FluidForms = [
+      'Ampulle(n)',
+      'Beutel',
+      'Bolus/Boli',
+      'Bq',
+      'Dose(n)',
+      'Durchstechflasche(n)',
+      'Einmaldosenbehälter',
+      'Einzeldose(n)',
+      'Fertigspritze',
+      'Fertigspritze(n)',
+      'Flasche(n)',
+      'I.E.',
+      'Infusionskonzentrat',
+      'Infusionslösung',
+      'Inhalationen',
+      'Inhalator',
+      'Injektions-Set',
+      'Injektions-Sets',
+      'Injektor(en), vorgefüllt/Pen',
+      'Klistier(e)',
+      'MBq',
+      'Pipetten',
+      'Sachet(s)',
+      'Spritze(n)',
+      'Sprühstösse',
+      'Stechampulle (Lyophilisat) und Ampulle (Solvens)',
+      'Stechampulle',
+      'Suspension',
+      'Zylinderampulle(n)',
+      'cartouches',
+      'dose(s)',
+      'flacon perforable',
+      'sacchetto',
+      'vorgefüllter Injektor',
+      ]
+    FesteFormen = [
+      'Depotabs',
+      'Dragée(s)',
+      'Generator mit folgenden Aktivitäten:',
+      'Filmtabletten',
+      'Gerät',
+      'Kapsel(n)',
+      'Kautabletten',
+      'Lutschtabletten',
+      'Kugeln',
+      'Ovulum',
+      'Packung(en)',
+      'Pflaster',
+      'Schmelzfilme',
+      'Set',
+      'Strips',
+      'Stück',
+      'Suppositorien',
+      'Tablette(n)',
+      'Tüchlein',
+      'Urethrastab',
+      'Vaginalzäpfchen',
+      'comprimé',
+      'comprimé pelliculé',
+      'comprimés',
+      'comprimés à libération modifiée',
+      'comprimés à croquer sécables',
+      'imprägnierter Verband',
+      'magensaftresistente Filmtabletten',
+      'ovale Körper',
+      'tube(s)',
+    ]
+    Mesurements = [ 'g', 'kg', 'l', 'mg', 'ml', 'cm', 'GBq']
+    Others = ['Kombipackung', 'emballage combiné' ]
+    UnknownGalenicForm = 140
+    UnknownGalenicGroup = 1
+    Data_dir = File.expand_path(File.join(File.dirname(__FILE__),'..','..', 'data'))
+    @@galenic_groups  = YAML.load_file(File.join(Data_dir, 'gal_groups.yaml'))
+    @@galenic_forms   = YAML.load_file(File.join(Data_dir, 'gal_forms.yaml'))
+    @@new_galenic_forms = []
+    @@names_without_galenic_forms = []
+    @@rules_counter = {}
+    attr_accessor   :galenic_form, :unit, :pkg_size
+    attr_reader     :name, :substances, :composition
+    attr_reader     :selling_units, :count, :multi, :measure, :addition, :scale # s.a. commercial_form in oddb.org/src/model/part.rb
+    def self.get_galenic_group(name, lang = 'de')
+      @@galenic_groups.values.collect { |galenic_group|
+        return galenic_group if galenic_group.descriptions[lang].eql?(name)
+      }
+      @@galenic_groups[1]
+    end
+
+    def self.report_conversion
+      lines = [ '', '',
+                'Report of used conversion rules',
+                '-------------------------------',
+                ''
+                ]
+      @@rules_counter.each{
+        | key, value|
+          lines << "#{key}: #{value} occurrences"
+      }
+      lines << ''
+      lines << ''
+      lines
+    end
+
+    def self.get_galenic_form(name, lang = 'de')
+      @@galenic_forms.values.collect { |galenic_form|
+        return galenic_form if galenic_form.descriptions[lang].eql?(name)
+        if name and galenic_form.descriptions[lang].eql?(name.sub(' / ', '/'))
+          return galenic_form
+        end
+      }
+      @@galenic_forms[UnknownGalenicForm]
+    end
+
+    def self.dump_new_galenic_forms
+      if @@new_galenic_forms.size > 0
+        "\n\n\nAdded the following galenic_forms\n"+ @@new_galenic_forms.uniq.join("\n")
+      else
+        "\n\n\nNo new galenic forms added"
+      end
+    end
+    def self.dump_names_without_galenic_forms
+      if @@names_without_galenic_forms.size > 0
+        "\n\n\nThe following products did not have a galenic form in column Präparateliste\n"+ @@names_without_galenic_forms.sort.uniq.join("\n")
+      else
+        "\n\n\nColumn Präparateliste has everywhere a name\n"
+      end
+    end
+
+    def initialize(name = nil, size = nil, unit = nil, composition= nil)
+      @name = name
+      @pkg_size = size
+      @unit = unit
+      # @pkg_size, @galenic_group, @galenic_form =
+      search_galenic_info
+      @composition = composition
+      @measure = unit if unit and not @measure
+      @measure = @galenic_form.description if @galenic_form and not @measure
+      @galenic_form  ||= @@galenic_forms[UnknownGalenicForm]
+    end
+
+    def galenic_group
+      @@galenic_groups[@galenic_form.galenic_group]
+    end
+
+    # helper for generating csv
+    def headers
+      [ "name", "pkg_size", "selling_units", "measure",
+        # "count", "multi", "addition", "scale", "unit",
+        "galenic_form",
+        "galenic_group"
+        ]
+    end
+    def to_array
+      [ @name, @pkg_size, @selling_units, @measure,
+        # @count, @multi, @addition, @scale, @unit,
+        galenic_form  ? galenic_form.description  : '' ,
+        galenic_group ? galenic_group.description : ''
+        ]
+    end
+    def galenic_form__xxx
+      @galenic_form.description
+    end
+  private
+    def capitalize(string)
+      string.split(/\s+/u).collect { |word| word.capitalize }.join(' ')
+    end
+
+    def update_rule(rulename)
+      @@rules_counter[rulename] ||= 0
+      @@rules_counter[rulename] += 1
+    end
+
+    def get_selling_units(part_from_name_C, pkg_size_L, einheit_M)
+      begin
+        return pkg_size_to_int(pkg_size_L) unless part_from_name_C
+        part_from_name_C = part_from_name_C.gsub(/[()]/, '_')
+        FesteFormen.each{ |x|
+                          if part_from_name_C and (x.gsub(/[()]/, '_')).match(part_from_name_C)
+                            puts "feste_form in #{part_from_name_C} matched: #{x}" if $VERBOSE
+                            update_rule('feste_form name_C')
+                            @measure = x
+                            return pkg_size_to_int(pkg_size_L)
+                          end
+                          if einheit_M and x.eql?(einheit_M)
+                            puts "feste_form in einheit_M #{einheit_M} matched: #{x}" if $VERBOSE
+                            update_rule('feste_form einheit_M')
+                            @measure = x
+                            return pkg_size_to_int(pkg_size_L)
+                          end
+                        }
+        FluidForms.each{ |x|
+                          if part_from_name_C and x.match(part_from_name_C)
+                            puts "liquid_form in #{part_from_name_C} matched: #{x}" if $VERBOSE
+                            update_rule('liquid_form name_C')
+                            @measure = x
+                            return pkg_size_to_int(pkg_size_L, true)
+                          end
+                          if part_from_name_C and x.match(part_from_name_C.split(' ')[0])
+                            puts "liquid_form in #{part_from_name_C} matched: #{x}" if $VERBOSE
+                            update_rule('liquid_form first_part')
+                            return pkg_size_to_int(pkg_size_L, true)
+                          end
+                          if einheit_M and x.eql?(einheit_M)
+                            puts "liquid_form in einheit_M #{einheit_M} matched: #{x}" if $VERBOSE
+                            update_rule('liquid_form einheit_M')
+                            @measure = x
+                            return pkg_size_to_int(pkg_size_L, true)
+                          end
+                        }
+        Mesurements.each{ |x|
+                          if pkg_size_L and pkg_size_L.split(' ').index(x)
+                            puts "measurement in pkg_size_L #{pkg_size_L} matched: #{x}" if $VERBOSE
+                            update_rule('measurement pkg_size_L')
+                            @measure = x
+                            return pkg_size_to_int(pkg_size_L, true)
+                          end
+                          if einheit_M and /^#{x}$/i.match(einheit_M)
+                            puts "measurement in einheit_M #{einheit_M} matched: #{x}" if $VERBOSE
+                            update_rule('measurement einheit_M')
+                            @measure = x
+                            return pkg_size_to_int(pkg_size_L, true)
+                          end
+                        }
+        puts "Could not find anything for name_C #{part_from_name_C} pkg_size_L: #{pkg_size_L} einheit_M #{einheit_M}" if $VERBOSE
+        update_rule('unbekannt')
+        return 'unbekannt'
+      rescue RegexpError => e
+        puts "RegexpError for M: #{einheit_M} pkg_size_L #{pkg_size_L} C: #{part_from_name_C}"
+        update_rule('RegexpError')
+        return 'error'
+      end
+    end
+
+    def pkg_size_to_int(pkg_size, skip_last_part = false)
+      return pkg_size if pkg_size.is_a?(Fixnum)
+      return 1 unless pkg_size
+      parts = pkg_size.split(/\s*x\s*/i)
+      parts = parts[0..-2] if skip_last_part and parts.size > 1
+      last_multiplier = parts[-1].to_i > 0 ? parts[-1].to_i : 1
+      if parts.size == 3
+        return parts[0].to_i * parts[1].to_i * last_multiplier
+      elsif parts.size == 2
+        return parts[0].to_i * last_multiplier
+      elsif parts.size == 1
+        return last_multiplier
+      else
+        return 1
+      end
+    end
+    # Parse a string for a numerical value and unit, e.g. 1.5 ml
+    def self.check_for_value_and_units(what)
+      if m = /^([\d.]+)\s*(\D+)/.match(what)
+        # return [m[1], m[2] ]
+        return m[0].to_s
+      else
+        nil
+      end
+    end
+    def search_galenic_info
+      @substances = nil
+      @substances = @composition.split(/\s*,(?!\d|[^(]+\))\s*/u).collect { |name| capitalize(name) }.uniq if @composition
+
+      name = @name.clone
+      parts = name.split(',')
+      if parts.size == 1
+        @@names_without_galenic_forms << name
+      else
+        form_name = parts[-1].strip
+        @galenic_form = Calc.get_galenic_form(form_name)
+        # puts "oid #{UnknownGalenicForm} #{@galenic_form.oid} for #{name}"
+        if @galenic_form.oid == UnknownGalenicForm
+          @galenic_form =  GalenicForm.new(0, {'de' => form_name}, @@galenic_forms[UnknownGalenicForm] )
+          @@new_galenic_forms << form_name
+        end
+      end
+      @name = name.strip
+      if @pkg_size.is_a?(Fixnum)
+        @count = @pkg_size
+        res = [@pkg_size]
+      else
+        res = @pkg_size ? @pkg_size.split(/x/i) : []
+        if res.size >= 1
+          @count = res[0].to_i
+        else
+          @count = 1
+        end
+      end
+      # check whether we find numerical and units
+      if res.size >= 2
+        if (result = Calc.check_for_value_and_units(res[1].strip)) != nil
+          @multi = result[1].to_f
+        else
+          @multi = res[1].to_i
+        end
+      else
+        @multi = 1
+      end
+      @addition = 0
+      @scale = 1
+      @selling_units = get_selling_units(form_name, @pkg_size, @unit)
+    end
+  end
+end