oddb2xml 2.0.5 → 2.0.6

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 4e7b29934682850002244797289099295579fb51
-  data.tar.gz: a3d02cc3c125315dc26938cb6ce77ec033f98561
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MDA5ZTJmMjQ1NzlhMWM0ZjE5YzY5NjA2NWFjNGFlYjQxMjI0NjIzYw==
+  data.tar.gz: !binary |-
+    MmUxZWYwYTBiYjkyMjdiMGMzM2QwZjQwOWI5MTZjZTIxMmQ2YzhhYw==
 SHA512:
-  metadata.gz: 035cfd64bf3ffffa1a1e0b99c41331326dbb4cf181033c73491c89d81a785f48033e269cd004a262ba646996374c2807229960a140b9b839dca9c2adca528995
-  data.tar.gz: 2f50567e9e4d2e84e1bbf1ca34ae185d1b7b91f323772bde33e29059e56d7d99a2d552d8e40dc1f2d15a0eb7143d60db64d8ace9caf6b42641f1340437c70391
+  metadata.gz: !binary |-
+    MDBiZDcyMzQyZTMzMDhiNDFmNGQxOTAwOTc2MjA1OTY4NjMyNzhiNWE3NzI4
+    ODVlM2RmMTg2MTRkODdjODhlM2ExNDVkOTUyOTgwZTM3MzY1MzA1Y2Q2MmRi
+    NjI4Zjc0OTlkM2M0NTIzMmZlOTlmYzcwMGQ0YTdlMTc5NjZiNjY=
+  data.tar.gz: !binary |-
+    OGY4NThmMTRhNDVhY2ZlYzdmZmYwMThjMGY0NTM5MmJhYjMxYThjOTE4ZTUy
+    MmYwYzlhOWI5OTQ5OWQ0MDllMjk2YmM3OGY3YzUzMDE4MTM0YTYzOTRlY2Uz
+    NzM0NTQwZTlhOWQwZjMwZDlmMGYzMWM2ZmIzZDhlZjlmMDE1MjQ=

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    oddb2xml (2.0.5)
+    oddb2xml (2.0.6)
       archive-tar-minitar (~> 0.5.2)
       mechanize (~> 2.5.1)
       nokogiri (~> 1.5.10)
+      parslet (~> 1.7.0)
       rubyXL (~> 3.3.1)
       rubyzip (~> 1.1.3)
       savon (~> 2.4.0)
@@ -14,11 +15,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.3.6)
+    addressable (2.3.8)
     akami (1.2.2)
       gyoku (>= 0.4.0)
       nokogiri
     archive-tar-minitar (0.5.2)
+    blankslate (3.1.3)
     builder (3.2.2)
     coderay (1.1.0)
     columnize (0.9.0)
@@ -29,15 +31,16 @@ GEM
       debugger-linecache (~> 1.2.0)
       debugger-ruby_core_source (~> 1.3.5)
     debugger-linecache (1.2.0)
-    debugger-ruby_core_source (1.3.7)
+    debugger-ruby_core_source (1.3.8)
     diff-lcs (1.2.5)
-    domain_name (0.5.23)
+    domain_name (0.5.24)
       unf (>= 0.0.5, < 1.0.0)
     gyoku (1.1.1)
       builder (>= 2.1.2)
     httpi (2.1.1)
       rack
       rubyntlm (~> 0.3.2)
+    json (1.8.2)
     mechanize (2.5.1)
       domain_name (~> 0.5, >= 0.5.1)
       mime-types (~> 1.17, >= 1.17.2)
@@ -53,6 +56,8 @@ GEM
     nokogiri (1.5.11)
     nori (2.3.0)
     ntlm-http (0.1.1)
+    parslet (1.7.0)
+      blankslate (>= 2.0, <= 4.0)
     pry (0.10.1)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -63,20 +68,22 @@ GEM
     rack (1.6.0)
     rake (10.4.2)
     rdoc (4.2.0)
-    rspec (3.1.0)
-      rspec-core (~> 3.1.0)
-      rspec-expectations (~> 3.1.0)
-      rspec-mocks (~> 3.1.0)
-    rspec-core (3.1.7)
-      rspec-support (~> 3.1.0)
-    rspec-expectations (3.1.2)
+      json (~> 1.4)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.3)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.1.0)
-    rspec-mocks (3.1.3)
-      rspec-support (~> 3.1.0)
-    rspec-support (3.1.2)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
     ruby-ole (1.2.11.8)
-    rubyXL (3.3.6)
+    rubyXL (3.3.8)
       nokogiri (>= 1.4.4)
       rubyzip (>= 1.1.6)
     rubyntlm (0.3.4)
@@ -93,16 +100,16 @@ GEM
     sax-machine (0.1.0)
       nokogiri (> 0.0.0)
     slop (3.6.0)
-    spreadsheet (1.0.0)
+    spreadsheet (1.0.3)
       ruby-ole (>= 1.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.6)
+    unf_ext (0.0.7.1)
     wasabi (3.2.3)
       httpi (~> 2.0)
       mime-types (< 2.0.0)
       nokogiri (>= 1.4.0)
-    webmock (1.20.4)
+    webmock (1.21.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
     webrobots (0.1.1)

data/History.txt

@@ -1,3 +1,8 @@
+=== 2.0.6 / 27.04.2015
+
+* Remove trailing spaces in names imported from ZurRose transfer.dat
+* Use ISO-8859-14 when reading ZurRose transfer.dat to avoid unreadable characters, eg. Ethacridin
+
 === 2.0.5 / 31.03.2015
 
 * --calc adds boolean flag is_active_agent

data/dokumentation_calc.textile

@@ -0,0 +1,50 @@
+h1. Dokumentation für oddb2xml --calc
+
+Wenn man oddb2xml mit der Option @--calc@ aufruft, wird eine Datei @oddb_calc.xml@ erstellt, welche aufgrund diverser offener Datenquellen, (u.a. "Excel-Version Zugelassene Verpackungen":https://www.swissmedic.ch/arzneimittel/00156/00221/00222/00230/index.html?lang=de ) die Zusammensetzung aller in der Schweiz öffentlich zugelassener Medikamente im XML-Format erstellt.
+
+h2. Ziel
+
+Die Swissmedic verpackt in der Excel-Datei für die Packungsbsbeschreibung in den Spalte Q @Zusammensetzung@, eine Menge an Informationen, aus welchen Bestandteilen ein Medikament besteht. Zu einem grossen Teil folgt dies einer Syntax, für welche keine öffentlich zugängliche Dokumention zu bestehen scheint. Deshalb wurde für das Projekt ODDB.org anfangs 2015 beschlossen, die Analyse dieses Feldes von einem auf (was SW-Ingenieure) regulären Ausdrücken basierenden Algorithum auf einen echten Parser umzustellen.
+
+In diesem Dokument versuchen wir die dabei erkannten Elemente kurz zu dokumentieren und Grenzfälle und Entscheide anhand von Beispielen (mit IKSNR und Name identifiert) zu begründen.
+
+
+h2. Syntax
+
+Die Syntax wird in einer Datei "compositions_syntax":https://raw.githubusercontent.com/ngiger/oddb2xml/master/lib/oddb2xml/compositions_syntax.rb beschrieben.
+
+* ratio:	Falls eine Zeile in der Zusammensetzung sow was wie @ratio: 1:10@ oder @ratio: 1:1.5-2.4@ enthält, wird das Feld "more_info" entsprechend gesetzt.
+
+* Namen mit Zahlen. Beschloss dass @Glyceroli Monostearas 33-45@ eine Name und nicht Glyceroli Monostearas von 33 bis 45 von irgendwas ist.
+
+h2. Gebrauchte Abkürzungen und Schlüsselworte
+
+* q.s.
+* pro praeparatione
+* excipiens
+* pro compresso obducto
+* pro compresso
+* epro praeparatione
+* ad pulverem
+* pro charta
+* ad globulos
+* aqua ad iniectabilia q.s. ad solutionem
+* solvens (i.v.): aqua ad iniectabilia
+* ad solutionem
+* q.s. ad
+* aqua q.s. ad
+* saccharum ad
+* aether q.s.
+* aqua ad iniectabilia
+* q.s. pro praeparatione
+* ana partes
+* et oder @,@ trennen Substanzen
+* ut vorgehende Substanz wird via nachstehend aufgeführte Salze(e) aufgenommen
+
+
+h2. Vorgeschlag für Bereinigung
+
+* SwissmedicErrorHandler
+* Corresp: als Label für Bestandteile
+* corresp. für zugehörende Substanz
+* @<@ durch @max.@ ersetzen (1 Mal)

data/lib/oddb2xml/builder.rb

@@ -647,6 +647,10 @@ module Oddb2xml
             next unless row and row.cells[0] and row.cells[0].value and row.cells[0].value.to_i > 0
             iksnr               = "%05i" % row.cells[0].value.to_i
             seqnr               = "%02d" % row.cells[1].value.to_i
+            if row_nr % 250 == 0
+                puts "#{Time.now}: At row #{row_nr} iksnr #{iksnr}";
+                $stdout.sync
+            end
             no8                 = sprintf('%05d',row.cells[0].value.to_i) + sprintf('%03d',row.cells[10].value.to_i)
             name                = row.cells[2].value
             atc_code            = row.cells[5]  ? row.cells[5].value  : nil
@@ -680,12 +684,13 @@ module Oddb2xml
               xml.COMPOSITIONS {
                 info.compositions.each { |composition|
                   xml.COMPOSITION {
-                    # xml.SOURCE composition.source # emit this if you want to debug the results
+                    xml.CORRESP composition.corresp if composition.corresp
                     xml.LABEL composition.label if composition.label
                     xml.LABEL_DESCRIPTION composition.label_description if composition.label_description
                     xml.SUBSTANCES {
                       composition.substances.each { |substance|
                         xml.SUBSTANCE {
+                          xml.MORE_INFO substance.more_info.gsub('&gt','>').gsub('&amp', '&') if substance.more_info
                           xml.SUBSTANCE_NAME substance.name
                           xml.IS_ACTIVE_AGENT substance.is_active_agent
                           if substance.unit
@@ -693,9 +698,9 @@ module Oddb2xml
                             xml.UNIT substance.unit
                           end
                           if substance.chemical_substance
-                            xml.CHEMICAL_SUBSTANCE  substance.chemical_substance
-                            xml.CHEMICAL_QTY  substance.chemical_qty
-                            xml.CHEMICAL_UNIT substance.chemical_unit
+                            xml.CHEMICAL_SUBSTANCE  substance.chemical_substance.name
+                            xml.CHEMICAL_QTY        substance.chemical_substance.qty
+                            xml.CHEMICAL_UNIT       substance.chemical_substance.unit
                           end
                         }
                       }

data/lib/oddb2xml/calc.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 require 'oddb2xml/util'
-require 'oddb2xml/parse_compositions'
+require 'oddb2xml/parslet_compositions'
 require 'yaml'
 
 module Oddb2xml

data/lib/oddb2xml/cli.rb

@@ -330,6 +330,7 @@ module Oddb2xml
         lines << Calc.dump_new_galenic_forms
         lines << Calc.dump_names_without_galenic_forms
         lines << Calc.report_conversion
+        lines << ParseComposition.report
       end
       unless @options[:address]
         LANGUAGES.each do |lang|

data/lib/oddb2xml/compositions_syntax.rb

@@ -0,0 +1,368 @@
+# encoding: utf-8
+
+# This file is shared since oddb2xml 2.0.0 (lib/oddb2xml/parse_compositions.rb)
+# with oddb.org src/plugin/parse_compositions.rb
+#
+# It allows an easy parsing of the column P Zusammensetzung of the swissmedic packages.xlsx file
+#
+
+require 'parslet'
+require 'parslet/convenience'
+include Parslet
+
+class CompositionParser < Parslet::Parser
+
+  # Single character rules
+  rule(:lparen)     { str('(') }
+  rule(:rparen)     { str(')') }
+  rule(:comma)      { str(',') }
+
+  rule(:space)      { match('\s').repeat(1) }
+  rule(:space?)     { space.maybe }
+
+  # Things
+  rule(:digit) { match('[0-9]') }
+  rule(:digits) { digit.repeat(1) }
+  rule(:number) {
+    (
+      str('-').maybe >> (
+        str('0') | (match('[1-9]') >> match('[0-9\']').repeat)
+      ) >> (
+            (str('*') >>  digit.repeat(1)).maybe >>
+            (match(['.,^']) >> digit.repeat(1)).repeat(1)
+      ).maybe >> (
+        match('[eE]') >> (str('+') | str('-')).maybe >> digit.repeat(1)
+      ).maybe
+    )
+  }
+  rule(:radio_isotop) { match['a-zA-Z'].repeat(1) >> lparen >> digits >> str('-') >> match['a-zA-Z'].repeat(1-3) >> rparen >>
+                        ((space? >> match['a-zA-Z']).repeat(1)).repeat(0)
+                        } # e.g. Xenonum (133-Xe) or yttrii(90-Y) chloridum zum Kalibrierungszeitpunkt
+  rule(:ratio_value) { match['0-9:\-\.'].repeat(1)  >> space?}  # eg. ratio: 1:1, ratio: 1:1.5-2.4., ratio: 1:0.68-0.95
+
+  # handle stuff like acidum 9,11-linolicum or 2,2'-methylen-bis(6-tert.-butyl-4-methyl-phenolum) specially. it must contain at least one a-z
+  rule(:umlaut) { match(['éàèèçïöäüâ']) }
+  rule(:identifier_D12) { match['a-zA-Z'] >>  match['0-9'].repeat(1) }
+  rule(:identifier)  {  str('A + B') | str('ethanol.') | str('poloxamerum 238') | str('TM:') | str('&') | # TODO: why do we have to hard code these identifiers?
+                        str('F.E.I.B.A.') | str('LA 25% TM') | str('50/50') | str('polysorbatum ') >> digit >> digit | str('q.s.') |
+                        digit >> digit.maybe >> space >> str('per centum ') >> str('q.s.').maybe| str('1g/9.6 cm²') |
+                        str('9 g/L 5.4 ml') |
+                        str('spag.') | str('spp.') | str('ssp.') | str('deklar.') | # TODO: Sind diese Abkürzung wirklich Teil eines Substanznamens?
+                        str('ca.') | str('var.') | str('spec.') |
+                        identifier_D12 | identifier_without_comma | identifier_with_comma
+                     }
+
+  rule(:identifier_with_comma) {
+    match['0-9,\-'].repeat(0) >> (match['a-zA-Z']|umlaut)  >> (match(['_,']).maybe >> (match['0-9a-zA-Z\-\'\/'] | umlaut)).repeat(0)
+  }
+
+  rule(:identifier_without_comma) {
+    match['0-9\',\-'].repeat(0) >> (match['a-zA-Z']|umlaut)  >> (match(['_']).maybe >> (match['0-9a-zA-Z\-\'\/'] | umlaut)).repeat(0) >>
+        lparen >> (rparen.absent? >> any).repeat(1) >> rparen
+  }
+  rule(:one_word) { match['a-zA-Z'] >> match['0-9'].repeat(1) | match['a-zA-Z'].repeat(1) }
+  rule(:in_parent) { lparen >> one_word.repeat(1) >> rparen }
+  rule(:words_nested) { one_word.repeat(1) >> in_parent.maybe >> space? >> one_word.repeat(0) }
+  # dose
+  # 150 U.I. hFSH et 150 U.I. hLH
+  rule(:dose_unit)      { (str('cm²') |
+                           str('g/dm²') |
+                           str('g/l') |
+                           str('g/L') |
+                           str('% V/V') |
+                           str('µg/24 h') |
+                           str('µg/g') |
+                           str('µg') |
+                           str('ng') |
+                           str('guttae') |
+                           str('mg/g') |
+                           str('mg/ml') |
+                           str('MBq/ml') |
+                           str('MBq') |
+                           str('CFU') |
+                           str('mg') |
+                           str('Mg') |
+                           str('kJ') |
+                           str('G') |
+                           str('g') |
+                           str('l') |
+                           str('µl') |
+                           str('U. Ph. Eur.') |
+                           str('ml') |
+                           str('µmol') |
+                           str('mmol/l') |
+                           str('mmol') |
+                           str('Mio CFU') |
+                           str('Mio U.I.') |
+                           str('Mio U.') |
+                           str('Mio. U.I.') |
+                           str('Mio. U.') |
+                           str('Mia. U.I.') |
+                           str('Mia. U.') |
+                           str('U. Botox,') | # TODO: Should be U. Botox
+                           str('U.I. hFSH') |
+                           str('U.I. hCG') |
+                           str('U.I. hLH') |
+                           str('U.I.') |
+                           str('U./ml') |
+                           str('U.') |
+                           str('Mia.') |
+                           str('Mrd.') |
+                           str('% m/m') |
+                           str('% m/m') |
+                           str('%')
+                          ).as(:unit) }
+  rule(:qty_range)       { (number >> space? >> (str('+/-') | str(' - ') | str(' -') | str('-') | str('±') ) >> space? >> number).as(:qty_range) }
+  rule(:qty_unit)       { dose_qty >> (space >> dose_unit).maybe }
+  rule(:dose_qty)       { number.as(:qty) }
+  rule(:min_max)        { str('mind.') | (str('min.') | str('max.') | str('ca.') | str('<') ) >> space? } # TODO: swissmedic should replace mind. -> min.
+  # 75 U.I. hFSH et 75 U.I. hLH
+  rule(:dose_fsh) { qty_unit >> space >> str('et') >> space >> qty_unit.as(:dose_right) }
+  rule(:dose_per) { (digits >> str('/') >> digits).as(:qty)}
+  rule(:dose)           { dose_fsh |
+                          dose_per |
+                          ( min_max.maybe >>
+                            ( (qty_range >> (space >> dose_unit).maybe) | (qty_unit | dose_qty |dose_unit)) >> space? )
+                           }
+  rule(:dose_with_unit) { min_max.maybe >>
+                            dose_fsh |
+                          ( qty_range >> space >> dose_unit |
+                            dose_qty  >> space >> dose_unit
+                          ) >>
+                          space?
+                        }
+  rule(:operator)   { match('[+]') >> space? }
+
+  # Grammar parts
+  rule(:useage) {   (any >> str('berzug:')) | # match Überzug
+                    str('antiox.:') |
+                    str('arom.:') |
+                    str('conserv.:') |
+                    str('color.:')
+                   }
+  rule(:lebensmittel_zusatz) {  str('E').as(:lebensmittel_zusatz) >> space >>
+                                (digits >> match['(a-z)'].repeat(0,3)).as(:digits) >>
+                                (space >> dose.as(:dose_lebensmittel_zusatz)).maybe >> space?
+
+                   } # Match Wirkstoffe like E 270
+  rule(:der) { (str('DER:')  >> space >> digit >> match['0-9\.\-:'].repeat).as(:der) >> space?
+             } # DER: 1:4 or DER: 3.5:1 or DER: 6-8:1 or DER: 4.0-9.0:1'
+  rule(:forbidden_in_substance_name) {
+                           useage |
+                           min_max |
+                           str('corresp. ca.,') |
+                           str(', corresp.') |
+                           str('corresp.') |
+                           str('ratio:') |
+                            str('Mio ') |
+                            str('et ') |
+                            str('ut ') |
+                            str('Beutel: ') |
+                            str('ut alia: ') |
+                            str('pro dosi') |
+                            str('pro capsula') |
+                            str('pro vitroe') |
+                            (digits.repeat(1) >> space >> str(':')) | # match 50 %
+                            str('ad globulos') |
+                            str('ana ') |
+                            str('ana partes') |
+                            str('partes') |
+                            str('ad pulverem') |
+                            str('ad suspensionem') |
+                            str('q.s. ad ') |
+                            str('q.s. pro ') |
+                            str('ad solutionem') |
+                            str('ad emulsionem') |
+                            str('excipiens')
+    }
+  rule(:name_without_parenthesis) {
+    (
+      (str('(') | forbidden_in_substance_name).absent? >>
+        (radio_isotop | str('> 1000') | str('> 500') | identifier.repeat(1)) >>
+      space?
+    ).repeat(1)
+  }
+
+  rule(:part_with_parenthesis) { lparen >> ( (lparen | rparen).absent? >> any).repeat(1) >>
+                                 (part_with_parenthesis | rparen >> str('-like:') | rparen  ) >> space?
+                               }
+  rule(:name_with_parenthesis) {
+    forbidden_in_substance_name.absent? >>
+    ((comma | lparen).absent? >> any).repeat(0) >> part_with_parenthesis >>
+    (forbidden_in_substance_name.absent? >> (identifier.repeat(1) | part_with_parenthesis | rparen) >> space?).repeat(0)
+  }
+  rule(:substance_name) { (
+                            der |
+                            name_with_parenthesis |
+                            name_without_parenthesis
+                          ) >>
+                          str('pro dosi').maybe >> space?
+                          }
+  rule(:simple_substance) { substance_name.as(:substance_name) >> space? >> dose.as(:dose).maybe}
+  rule(:simple_subtance_with_digits_in_name_and_dose)  {
+    substance_lead.maybe >> space? >>
+    (name_without_parenthesis >> space? >> ((digits.repeat(1) >> (str(' %') | str('%')) | digits.repeat(1)))).as(:substance_name) >>
+    space >> dose_with_unit.as(:dose)
+  }
+
+
+  rule(:pro_dose) { str('pro') >>  space >> dose.as(:dose_corresp) }
+
+    # TODO: what does ut alia: impl?
+  rule(:substance_ut) {
+      (substance_lead.maybe >> simple_substance).as(:substance_ut) >>
+  (space? >> (str('pro dosi ut ') | str('ut ') )  >>
+    space? >> str('alia:').absent? >>
+    (excipiens |
+    substance_name >> space? >> str('corresp.') >> space? >> substance_lead.maybe >> space? >> simple_substance |
+    simple_substance
+     ).as(:for_ut)
+  ).repeat(1) >>
+    space? # >> str('alia:').maybe >> space?
+    }
+
+  rule(:substance_more_info) { # e.g. "acari allergeni extractum 5000 U.:
+      (str('ratio:').absent? >> (identifier|digits) >> space?).repeat(1).as(:more_info) >> space? >> (str('U.:') | str(':')| str('.:')) >> space?
+    }
+
+  rule(:dose_pro) { (
+                       str('excipiens ad solutionem pro ') |
+                       str('aqua q.s. ad gelatume pro ') |
+                       str('aqua q.s. ad solutionem pro ') |
+                       str('aqua q.s. ad suspensionem pro ') |
+                       str('q.s. ad pulverem pro ') |
+                       str('doses pro vase ') |
+                       str('pro vase ') |
+                       str('excipiens ad emulsionem pro ') |
+                       str('excipiens ad pulverem pro ') |
+                       str('aqua ad iniectabilia q.s. ad solutionem pro ')
+                    )  >> dose.as(:dose_pro) >> space? >> ratio.as(:ratio).maybe
+  }
+
+  rule(:excipiens)  { (dose_pro |
+                       str('excipiens pro compresso obducto') |
+                       str('excipiens pro compresso') |
+                       str('excipiens pro praeparatione') |
+                       str('excipiens') |
+                       str('ad pulverem') |
+                       str('pro charta') |
+                       str('ad globulos') |
+                       str('aqua ad iniectabilia q.s. ad solutionem') |
+                       str('solvens (i.v.): aqua ad iniectabilia') |
+                       str('ad solutionem') |
+                       str('q.s. ad') |
+                       str('aqua q.s. ad') |
+                       str('saccharum ad') |
+                       str('aether q.s.') |
+                       str('pro vitro') |
+                       str('aqua ad iniectabilia') |
+                       str('pro praeparatione') |
+                       str('q.s. pro praeparatione') |
+                       str('ana partes')
+                      ) >> space? >>
+                      ( any.repeat(0) )
+                      }
+
+  rule(:substance_lead) { useage.as(:more_info) >> space? |
+                      str('Beutel:').as(:more_info) >> space? |
+                      str('residui:').as(:more_info) >> space? |
+                      str('mineralia').as(:mineralia) >> str(':') >> space? |
+                      str('Solvens:').as(:solvens) >> space? |
+                      substance_more_info
+    }
+  rule(:corresp_substance_label) {
+      str(', corresp. ca.,') |
+      str('corresp. ca.,') |
+      str('corresp.') |
+      str('corresp., ') |
+      str(', corresp.')
+    }
+
+  rule(:corresp_substance) {
+                            (corresp_substance_label) >> space? >>
+                            (
+                             simple_substance.as(:substance_corresp) |
+                             dose.as(:dose_corresp_2)
+                            )
+  }
+
+  rule(:ratio) { str('ratio:') >>  space >> ratio_value }
+
+  rule(:solvens) { (str('Solvens:') | str('Solvens (i.m.):'))>> space >> (any.repeat).as(:solvens) >> space? >>
+                   (substance.as(:substance) >> str('/L').maybe).maybe  >>
+                    any.maybe
+                }
+  rule(:substance) {
+    simple_subtance_with_digits_in_name_and_dose |
+    useage.as(:more_info) >> space? >> excipiens |
+    ratio.as(:ratio) |
+    solvens |
+    der  >> corresp_substance.maybe |
+    (str('potenziert mit:') >> space).maybe >> excipiens.as(:excipiens) |
+    substance_ut |
+    substance_lead.maybe >> space? >> lebensmittel_zusatz |
+    substance_lead.maybe >> space? >> simple_substance >> corresp_substance.maybe >> space? >> corresp_substance.maybe >> space? >> dose_pro.maybe >> str('pro dosi').maybe
+  }
+  rule(:histamin) { str('U = Histamin Equivalent Prick').as(:histamin) }
+  rule(:praeparatio){ ((one_word >> space?).repeat(1).as(:description) >> str(':') >> space?).maybe >>
+                      (name_with_parenthesis | name_without_parenthesis).repeat(1).as(:substance_name) >>
+                      number.as(:qty) >> space >> str('U.:') >> space? >>
+                      ((identifier >> space?).repeat(1).as(:more_info) >> space?).maybe
+                    }
+  rule(:substance_separator) { (str(', et ') | comma | str('et ') | str('ut alia: ')) >> space? }
+  rule(:one_substance)       { (praeparatio | histamin | substance).as(:substance) >> space? >> ratio.as(:ratio).maybe }
+  # rule(:one_substance)       { (substance_ut).as(:substance) } # >> str('.').maybe }
+  rule(:all_substances)      { (one_substance >> substance_separator.maybe).repeat(1) }
+  rule(:composition)         { all_substances }
+  rule(:long_labels) {
+        str('Praeparatio sicca cum solvens: praeparatio sicca:') |
+        str('Praeparatio cryodesiccata') >> (str(':').absent? >> any).repeat(0) >> str(':') |
+        str('Tela cum praeparatione (Panel ') >> digit >> str('):')
+      }
+  rule(:label_id) {
+     (
+        str('V') |
+        str('IV') |
+        str('III') |
+        str('II') |
+        str('I') |
+        str('A') |
+        str('B') |
+        str('C') |
+        str('D') |
+        str('E')
+     )
+  }
+  rule(:label_separator) {  (str('):')  | str(')')) }
+  rule(:label) { label_id.as(:label) >> space? >>
+    label_separator >> str(',').absent?  >>
+               (space? >> (match(/[^:]/).repeat(0)).as(:label_description)  >> str(':') >> space).maybe
+  }
+  rule(:leading_label) {    label_id >> label_separator >> (str(' et ') | str(', ') | str(' pro usu: ') | space) >>
+                            label_id >> label_separator >> any.repeat(1)  |
+                            long_labels.as(:label) |
+                            label
+    }
+  rule(:corresp_label) {
+    str('doses ') |
+    str('Pulver: ') |
+    str('Diluens: ') |
+    str('Solvens (i.v.): ') |
+    str('Solvens (i.m.): ') |
+    str('Solvens: ') |
+    str('Solutio reconstituta:') |
+    str('Corresp., ') |
+    str('Corresp. ') |
+    str('corresp. ')
+  }
+  rule(:corresp_line) { corresp_label >> any.repeat(1).as(:corresp)  |
+                        ((label_id >> label_separator >> space? >> str('et ').maybe).repeat(1) >> any.repeat(1)).as(:corresp)
+  }
+
+  rule(:expression_comp) {
+    leading_label.maybe >> space? >> composition.as(:composition) >> space? >> str('.').maybe >> space? |
+    corresp_line
+  }
+  root :expression_comp
+end
+

data/lib/oddb2xml/extractor.rb

@@ -456,16 +456,25 @@ module Oddb2xml
     # see http://dev.ywesee.com/Bbmb/TransferDat
     def initialize(dat, extended = false)
       @@extended = extended
-      @@error_file ||= File.open(File.join(WorkDir, "duplicate_ean13_from_zur_rose.txt"), 'w+')
+      @@error_file ||= File.open(File.join(WorkDir, "duplicate_ean13_from_zur_rose.txt"), 'wb+:ISO-8859-14')
       @@items_without_ean13s ||= 0
       @@duplicated_ean13s ||= 0
       @@zur_rose_items ||= 0
-      @io = StringIO.new(dat) if dat
+      if dat
+        if File.exists?(dat)
+          @io = File.open(dat, 'rb:ISO-8859-14')
+        else
+          @io = StringIO.new(dat)
+        end
+        @io
+      else
+         nil
+      end
     end
     def to_hash
       data = {}
       while line = @io.gets
-        line = line.chomp
+        line = line.encode('utf-8').gsub("\u0089", "‰").gsub("\u0092", '’').gsub("\u0096", '-').chomp
         next if line =~ /(ad us\.* vet)|(\(vet\))/i
         if @@extended
           next unless line =~ /(\d{13})(\d{1})$/
@@ -490,7 +499,7 @@ module Oddb2xml
           :line   => line.chomp,
           :ean   => ean13,
           :vat   => line[96],
-          :description => line[10..59], # .sub(/\s+$/, ''),
+          :description => line[10..59].sub(/\s+$/, ''),
           :additional_desc => '',
           :pharmacode => pharma_code,
           :price => sprintf("%.2f", line[60,6].gsub(/(\d{2})$/, '.\1').to_f),