food_ingredient_parser 1.1.3 → 1.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 77503a77f269805d23ecb60a7b9d63401063e140
-  data.tar.gz: 412d0c6aab0924371677cb8e2a4201349f4684a8
+SHA256:
+  metadata.gz: a509e86303484d262ce6fd2d52b4db47d661e45ccbb7691e479e5257bd6c17a2
+  data.tar.gz: a1f50b40110cc4ba87e54ee410a3016d89796277882ceec3696a1e70d081c5f3
 SHA512:
-  metadata.gz: 82ba24e9277917326348e769feda731b030a9f6477f811f067b1ff1377e784d206adc56407a607e5b3498ddbbfcc8751c7ef6b3583490f59fa56771bdd9f0a80
-  data.tar.gz: '0768160047a661cad810a4fda0c7922e03b039bd17f4aa7b2022722cb636080fa01690a731da8d088861c6981163803b41d2bd577093f4b0a91250d4fe68c03d'
+  metadata.gz: 62424433ad156e03a9448e5250459bb9409e8111a41fd9e02db823fd2e5f26cee9954476c0d177eb0375fa5405dd1c4a968d652ae63ff9abf97f3599fc950a4f
+  data.tar.gz: 06a05663e658643253184589f621c4af2d12c51bf8825429d56480be187fb2ff2d07c956e5d8520e4cce377b1894c23d52dab7ba0644e1614276e7b3e87e0ca6

data/bin/food_ingredient_parser

@@ -41,8 +41,10 @@ def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false
   if parsed
     puts(parsed.inspect) if verbosity > 1
     pp(parsed.to_h, color: color) if verbosity > 0
+    return true
   else
     puts "(no result: #{parser.parser.failure_reason})" if verbosity > 0
+    return false
   end
 end
 
@@ -63,6 +65,7 @@ def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: fa
   pct_parsed   = 100.0 * count_parsed / (count_parsed + count_noresult)
   pct_noresult = 100.0 * count_noresult / (count_parsed + count_noresult)
   puts "parsed #{colorize(color && "1;32", count_parsed)} (#{pct_parsed.round(1)}%), no result #{colorize(color && "1;31", count_noresult)} (#{pct_noresult.round(1)}%)"
+  return count_noresult
 end
 
 verbosity = 1
@@ -108,8 +111,10 @@ if strings.any? || files.any?
     STDERR.puts("Please specify one of the known parsers: #{PARSERS.keys.join(", ")}.")
     exit(1)
   end
-  strings.each {|s| parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
-  files.each   {|f| parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
+  success = true
+  strings.each {|s| success &= parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
+  files.each   {|f| success &= parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) == 0 }
+  success or exit(1)
 else
   STDERR.puts("Please specify one or more --file or --string arguments (see --help).")
 end

data/lib/food_ingredient_parser/cleaner.rb

@@ -2,13 +2,14 @@ module FoodIngredientParser
   module Cleaner
 
     def self.clean(s)
-      s.gsub!("\u00ad", "")             # strip soft hyphen
-      s.gsub!("\u0092", "'")            # windows-1252 apostrophe - https://stackoverflow.com/a/15564279/2866660
-      s.gsub!("aÄs", "aïs")             # encoding issue for maïs
-      s.gsub!("ï", "ï")                # encoding issue
-      s.gsub!("ë", "ë")                # encoding issue
-      s.gsub!(/\A\s*"(.*)"\s*\z/, '\1') # enclosing double quotation marks
-      s.gsub!(/\A\s*'(.*)'\s*\z/, '\1') # enclosing single quotation marks
+      s.gsub!(/(_x005f_|_)x000d_/i, "\n") # fix sometimes encoding for newline
+      s.gsub!("\u00ad", "")               # strip soft hyphen
+      s.gsub!("\u0092", "'")              # windows-1252 apostrophe - https://stackoverflow.com/a/15564279/2866660
+      s.gsub!("‚", ",")                   # normalize unicode comma
+      s.gsub!("aÄs", "aïs")               # encoding issue for maïs
+      s.gsub!("ï", "ï")                  # encoding issue
+      s.gsub!("ë", "ë")                  # encoding issue
+      s.gsub!(/\A\s*(["']+)(.*)\1\s*\z/, '\2') # enclosing quotation marks
       s
     end
 

data/lib/food_ingredient_parser/loose/scanner.rb

@@ -4,19 +4,23 @@ module FoodIngredientParser::Loose
   class Scanner
 
     SEP_CHARS  = "|;,.".freeze
-    MARK_CHARS = "¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº⁽⁾†‡•°#^*".freeze
+    MARK_CHARS = "¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº⁽⁾†‡⁺•°▪◊#^˄*~".freeze
     PREFIX_RE  = /\A\s*(ingredients|contains|ingred[iï][eë]nt(en)?(declaratie)?|bevat|dit zit er\s?in|samenstelling|zutaten)\b\s*[:;.]?\s*/i.freeze
-    NOTE_RE    = /\A\b(dit product kan\b|deze verpakking kan\b|kan sporen\b.*?\bbevatten\b|voor allergenen\b|allergenen\b|E\s*=|gemaakt in\b|geproduceerd in\b|bevat mogelijk\b|kijk voor meer\b|allergie-info|in de fabriek\b|in dit bedrijf\b|voor [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bis [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bgebruikt\b)/i.freeze
+    NOTE_RE    = /\A\b(dit product kan\b|deze verpakking kan\b|kan sporen\b.*?\bbevatten\b|voor allergenen\b|allergenen\b|allergie[- ]informatie(\s*:|\b)|E\s*=|gemaakt in\b|geproduceerd in\b|bevat mogelijk\b|kijk voor meer\b|allergie-info|in de fabriek\b|in dit bedrijf\b|voor [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bis [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bgebruikt\b)/i.freeze
     # Keep in sync with +abbrev+ in the +Common+ grammar, plus relevant ones from the +Amount+ grammar.
     ABBREV_RE  = Regexp.union(
       /\A(
         N°\b |
         °C\b |
         (ijzer|chroom|koper)\s*\(I+\)\s*[[:alnum:]]+\b |
-        L\(\+\)-[[:alnum:]]+\b |
+        L\(\+\)[ -][[:alnum:]]+\b |
+        L\.\s+rhamnosus\b | L\.\s+acidophilus\b | L\.\s+casei\b | B\.\s+lactis | A\.\s+oryzae |
+        S\.\s+thermophilus\b | L\.\sbulgaricus\b |
+        T\.\s*aestivum\b(\s+vitt\.)? |
+        nucifera\s+L\. |
         type\s+"\d+" |
-        L\.\s+rhamnosus\b | L\.\s+acidophilus\b | L\.\s+casei\b | B\.\s+lactis | A\.\s+oryzae
-        E-e?\d{3}[a-z]?\s*\(i+\)
+        E-e?\d{3}[a-z]?\s*\(i+\) |
+        www\.[-_\/:%.A-Za-z0-9]+
       )/xi,
       *%w[
         a.o.p b.g.a b.o.b c.a c.i d.e d.m.v d.o.c d.o.p d.s e.a e.g e.u f.i.l f.o.s i.a

data/lib/food_ingredient_parser/loose/transform/amount_from_name.treetop

@@ -5,8 +5,9 @@ module FoodIngredientParser::Loose::Transform
 
     rule amount_from_name
       # just amount, amount in front or at the end
-      ws* amount:amount ws+ name:(.*) /
-      ws* amount:amount ws* /
+      ws* amount:amount                ws+ name:(.*) /
+      ws* amount:amount_simple_percent ws* name:(.*) /
+      ws* amount:amount                ws* /
       ws* name:( !amount word ( ws+ !amount word )* )+ ws* amount:amount ws*
     end
   end

data/lib/food_ingredient_parser/strict/grammar/amount.treetop

@@ -9,6 +9,10 @@ module FoodIngredientParser::Strict::Grammar
       amount:amount_simple                 <AmountNode>
     end
 
+    rule amount_simple_percent
+      amount:(amount_simple_number ws* percent) <AmountNode>
+    end
+
     rule amount_simple
       ( (
         'of which'i / 'at least'i / 'minimal'i / 'maximal'i / 'less than'i / 'more than'i /
@@ -17,23 +21,30 @@ module FoodIngredientParser::Strict::Grammar
       ) ws* )?
       amount_simple_quantity
       ( ws+ (
-        'of'i / 'or less of'i / 'or more of'i /
-        'minimaal'i / 'minimum'i / 'van het uitlekgewicht'i / 'van het geheel'i /
+        'of a'i / 'of'i / 'or less of'i / 'or more of'i /
+        'van een'i / 'minimaal'i / 'minimum'i / 'van het uitlekgewicht'i / 'van het geheel'i /
         'min.'i / 'min'i / 'max.'i / 'max'i
       ) )?
     end
 
     rule amount_simple_quantity
-      amount_simple_number ( ws* amount_simple_unit? ws* dash ws* amount_simple_number )? ws* amount_simple_unit
+      amount_simple_number ( ws* amount_simple_unit? ws* dash ws* amount_simple_number )? ( ws* amount_simple_unit )?
     end
 
     rule amount_simple_number
-      ( [±∓~∼∽≂≃≈≲≤<>≥≳] ws* )? number
+      ( amount_simple_comparator ws* )? number
+    end
+
+    rule amount_simple_comparator
+      '=' ws* [<>] /
+      [<>] ws* ( '=' / 'of gelijk aan'i !char / 'or equal to'i !char ) /
+      [±∓~∼∽≂≃≈≲≤<>≥≳] / '+/-' / '-/+'
     end
 
     rule amount_simple_unit
-      ( percent / ( ( 'procent' / 'percent' / 'gram'i / 'ml'i / 'mg'i / 'gr'i / 'g'i ) !char ) )
+      ( percent / ( ( 'procent' / 'percent' / 'gram'i / 'ml'i / 'mg'i / 'gr'i / 'g'i / 'ppm'i ) !char ) )
       ( ws 'vol'i ( !char / '.' ) )?
+      ( ws* '℮' )?
     end
   end
 end

data/lib/food_ingredient_parser/strict/grammar/common.treetop

@@ -10,17 +10,22 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule char
-      [[:alnum:]] /
+      !mark [[:alnum:]] /
       fraction /
-      [-/\`'´’+=_{}&] /
-      [®™] /
-      [¿?] /                         # weird characters turning up in names (e.g. encoding issues)
+      [-/\`'"´‘’+=_{}&] /
+      [®©™♣] /
+      [¿?¯] /                        # weird characters turning up in names (e.g. encoding issues)
       [₁₂₃₄₅₆₇₈₉]                    # can occur with vitamins
     end
 
     rule mark
       # mark referencing a footnote
-      [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾'? / '⁽' [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾' / [†‡•°#^] / '*'+ / '(' ws* ( [†‡•°#^] / '*'+ ) ws* ')'
+      [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾'? /
+      '⁽' [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾' /
+      [˄^] digit /
+      [†‡⁺•°▪◊#˄^~˛] /
+      '*'+ /
+      '(' ws* ( [†‡⁺•°▪◊#˄^~˛] / '*'+ ) ws* ')'
     end
 
     rule digit
@@ -28,7 +33,8 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule fraction
-      [½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅐⅛⅜⅝⅞⅑⅒]
+      [½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅐⅛⅜⅝⅞⅑⅒] /
+      digit+ '/' digit+
     end
 
     rule percent
@@ -51,6 +57,25 @@ module FoodIngredientParser::Strict::Grammar
       ( 'and' / 'en' / 'und' ) !char / '&'
     end
 
+    rule e_number
+      ( 'E'i '-'? [0-9] [0-9] [0-9] [[:alpha:]]? )
+      ( ( ws* '(' 'i'i+ ')' ) / ![[:alnum:]] ) # e.g. "E450 (iii)"
+    end
+
+    rule chem_systematic_name
+      ( chem_systematic_name_num dash ) ( chem_systematic_name_word dash chem_systematic_name_num dash ws? )* chem_systematic_name_word /
+      ( chem_systematic_name_word dash chem_systematic_name_num dash ws? )+ chem_systematic_name_word
+    end
+
+    rule chem_systematic_name_word
+      [A-Za-z]+ ( dash [A-Za-z]+ dash [A-Za-z]+ )*
+    end
+
+    rule chem_systematic_name_num
+      digit+ [RH] /
+      digit+ ( ',' digit+ )* '\''?
+    end
+
     rule abbrev
       # These are listed explicitely to avoid incorrect interpretations, and allow missing trailing dots.
       # To get an idea of what occurs (second one omits trailing dots):
@@ -105,12 +130,14 @@ module FoodIngredientParser::Strict::Grammar
         'w.o'i /
         'w.v'i /
         # not auto-generated additions
+        'nr.'i /
         'vit'i /   # vitamin
         'denat'i / # denaturated
         'alc'i /   # alcohol
         'vol'i /   # volume
         'conc'i /  # concentration
-        'subsp'i   # subspecies
+        'subsp'i / # subspecies
+        'www.'i [-_\/:%.A-Za-z0-9]+
       )
       '.'? ![[:alpha:]]
     end
@@ -121,10 +148,14 @@ module FoodIngredientParser::Strict::Grammar
         'N°'i /
         '°C'i /
         ( 'ijzer'i / 'chroom'i / 'koper'i ) ws* '(' 'I'i+ ')' ws* [[:alnum:]]+ /
-        'L(+)-' [[:alnum:]]+ /
+        'L(+)' ('-' / ws) [[:alnum:]]+ /
         'L.' ws+ 'rhamnosus'i / 'L.' ws+ 'acidophilus'i / 'L.' ws+ 'casei' / 'B.'i ws+ 'lactis'i / 'A.'i ws+ 'oryzae'i /
+        'S.' ws+ 'thermophilus'i / 'L.' ws+ 'bulgaricus'i /
+        'T.' ws* 'aestivum'i (ws+ 'vitt.'i)? /
+        'nucifera' ws+ 'L.'i /
         'type'i ws+ '"' [0-9]+ '"' /
-        'E'i '-'? [0-9] [0-9] [0-9] [[:alpha:]]? ws* '(' 'i'i+ ')' # e.g. "E450 (iii)"
+        e_number /
+        chem_systematic_name
       ) ![[:alpha:]]
     end
   end

data/lib/food_ingredient_parser/strict/grammar/ingredient_simple.treetop

@@ -9,13 +9,14 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule ingredient_simple_with_amount
-      pre:( '{' ws* )? amount:amount ws+ ing:ingredient_simple                   <IngredientNode> /
+      pre:( '{' ws* )? amount:amount                ws+ ing:ingredient_simple    <IngredientNode> /
+      pre:( '{' ws* )? amount:amount_simple_percent ws* ing:ingredient_simple    <IngredientNode> /
       ing:ingredient_simple ws* amount:amount post:( ws* '}' )? (ws? mark:mark)? <IngredientNode> /
       ing:ingredient_simple                                                      <IngredientNode>
     end
 
     rule ingredient_simple_e_number
-      name:( [Ee] [0-9] [0-9] [0-9] [a-zA-Z]? ) ![a-zA-Z0-9] <IngredientNode>
+      name:e_number <IngredientNode>
     end
 
   end

data/lib/food_ingredient_parser/strict/grammar/root.treetop

@@ -40,7 +40,7 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule root_mark_sentences_in_list
-      ( ( ws* [,.;] / ws ) ws* root_mark_sentence_in_list )+
+      ( ( ws* [,.;] / ws )+ root_mark_sentence_in_list )+
     end
 
     rule root_mark_sentence_in_list

data/lib/food_ingredient_parser/version.rb

@@ -1,4 +1,4 @@
 module FoodIngredientParser
-  VERSION      = '1.1.3'
-  VERSION_DATE = '2018-10-12'
+  VERSION      = '1.1.8'
+  VERSION_DATE = '2021-01-12'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: food_ingredient_parser
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.1.8
 platform: ruby
 authors:
 - wvengen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-12 00:00:00.000000000 Z
+date: 2021-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: treetop
@@ -87,8 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Parser for ingredient lists found on food products.