food_ingredient_parser 1.1.5 → 1.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8c80d036dbee183ed2fd1a8cc4e513e54318d142
-  data.tar.gz: 6cfcd29eacb9e99a9be9a66a90446f47f714ae4b
+SHA256:
+  metadata.gz: a56d22b7e67a3a913b051bcbda8da885ddd467dc53f5a0df0faa5b40759a1f35
+  data.tar.gz: 427dd79c9f9203dc7901ead6264e08c05183d02aec266ac1d3bff930a5ba1dcd
 SHA512:
-  metadata.gz: 75b1f91e5db6bcfcc24ad8eabe16b541663e5e344604c31b884ab1a5633c95dc045cdee2d384f5434f5143b778659cecd69917b9adb16e25341978380e486bcc
-  data.tar.gz: 7254cca971a558bda2ae6e996cc4d121fc5138f09d67c847bb44eec87421aeea39bf18a771137eb3b0b2bb734bed058a1d2347f815b9edc1b23e0b069d83a381
+  metadata.gz: 0b07032ade3a55ce208bcb0c069223b41aee21f185a2b6a9bb91332881dfef8e1d829ae966097e48ffdba9984517be43b10bd027099f9bdce04e3a4c6fc41ca8
+  data.tar.gz: ebdf452a09d54b151ce8cfa9bb65b4477dd1afc81bfc5cd1d94055f726d387f522dd04a3e11b73d4b26222a18bc1068912a81c8e2e3cd8439b0cee1c1ec290d7

data/README.md

@@ -69,6 +69,7 @@ Usage: bin/food_ingredient_parser [options] --file|-f <filename>
     -r, --parser PARSER              Use specific parser (strict, loose).
     -e, --[no-]escape                Escape newlines
     -c, --[no-]color                 Use color
+        --[no-]html                  Print as HTML with parsing markup
     -v, --[no-]verbose               Show more data (parsed tree).
         --version                    Show program version.
     -h, --help                       Show this help
@@ -103,6 +104,9 @@ RootNode+Root3 offset=0, "tomato" (contains,notes):
   SyntaxNode offset=6, ""
 {:contains=>[{:name=>"tomato"}]}
 
+$ bin/food_ingredient_parser --html -s "tomato"
+<div class="root"><span class='depth0'><span class='name'>tomato</span></span></div>
+
 $ food_ingredient_parser -v -r loose -s "tomato"
 "tomato"
 Node interval=0..5

data/bin/food_ingredient_parser

@@ -2,6 +2,7 @@
 #
 # Parser for food ingredient lists.
 #
+require 'cgi'
 require 'optparse'
 
 $:.push(File.expand_path(File.dirname(__FILE__) + "/../lib"))
@@ -31,22 +32,31 @@ def colorize(color, s)
   end
 end
 
-def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false, color: false)
+def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false, color: false, html: false)
   parsed ||= parser.parse(s)
 
   return unless print.nil? || (parsed && print == :parsed) || (!parsed && print == :noresult)
 
-  puts colorize(color && "0;32", escape ? s.gsub("\n", "\\n") : s) if verbosity > 0
+  puts colorize(color && "0;32", escape ? s.gsub("\n", "\\n") : s) if !html && verbosity > 0
 
-  if parsed
+  if !html && parsed
     puts(parsed.inspect) if verbosity > 1
     pp(parsed.to_h, color: color) if verbosity > 0
-  else
+
+  elsif !html && !parsed
     puts "(no result: #{parser.parser.failure_reason})" if verbosity > 0
+
+  elsif html && parsed
+    puts('<div class="root">' + parsed.to_html + '</div>') if verbosity > 0
+
+  else
+    puts('<div class="root">' + CGI.escapeHTML(parsed) + '</div>') if verbosity > 0
   end
+
+  return !!parsed
 end
 
-def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: false)
+def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: false, html: false)
   count_parsed = count_noresult = 0
   File.foreach(path) do |line|
     next if line =~ /^#/    # comment
@@ -57,12 +67,13 @@ def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: fa
     count_parsed += 1 if parsed
     count_noresult += 1 unless parsed
 
-    parse_single(line, parsed, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color)
+    parse_single(line, parsed, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color, html: html)
   end
 
   pct_parsed   = 100.0 * count_parsed / (count_parsed + count_noresult)
   pct_noresult = 100.0 * count_noresult / (count_parsed + count_noresult)
   puts "parsed #{colorize(color && "1;32", count_parsed)} (#{pct_parsed.round(1)}%), no result #{colorize(color && "1;31", count_noresult)} (#{pct_noresult.round(1)}%)"
+  return count_noresult
 end
 
 verbosity = 1
@@ -72,6 +83,7 @@ print = nil
 parser_name = :strict
 escape = false
 color = true
+html = false
 PARSERS = {
   strict: FoodIngredientParser::Strict::Parser,
   loose:  FoodIngredientParser::Loose::Parser
@@ -92,6 +104,7 @@ OptionParser.new do |opts|
   opts.on("-r", "--parser PARSER", "Use specific parser (#{PARSERS.keys.join(", ")}).") {|p| parser_name = p&.downcase&.to_sym }
   opts.on("-e", "--[no-]escape", "Escape newlines") {|e| escape = !!e }
   opts.on("-c", "--[no-]color", "Use color") {|e| color = !!e }
+  opts.on(      "--[no-]html", "Print as HTML with parsing markup") {|e| html = !!e }
   opts.on("-v", "--[no-]verbose", "Show more data (parsed tree).") {|v| verbosity = v ? 2 : 1 }
   opts.on(      "--version", "Show program version.") do
     puts("food_ingredient_parser v#{FoodIngredientParser::VERSION}")
@@ -108,8 +121,10 @@ if strings.any? || files.any?
     STDERR.puts("Please specify one of the known parsers: #{PARSERS.keys.join(", ")}.")
     exit(1)
   end
-  strings.each {|s| parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
-  files.each   {|f| parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
+  success = true
+  strings.each {|s| success &= parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color, html: html) }
+  files.each   {|f| success &= parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color, html: html) == 0 }
+  success or exit(1)
 else
   STDERR.puts("Please specify one or more --file or --string arguments (see --help).")
 end

data/lib/food_ingredient_parser/cleaner.rb

@@ -2,13 +2,14 @@ module FoodIngredientParser
   module Cleaner
 
     def self.clean(s)
-      s.gsub!("\u00ad", "")             # strip soft hyphen
-      s.gsub!("\u0092", "'")            # windows-1252 apostrophe - https://stackoverflow.com/a/15564279/2866660
-      s.gsub!("aÄs", "aïs")             # encoding issue for maïs
-      s.gsub!("ï", "ï")                # encoding issue
-      s.gsub!("ë", "ë")                # encoding issue
-      s.gsub!(/\A\s*"(.*)"\s*\z/, '\1') # enclosing double quotation marks
-      s.gsub!(/\A\s*'(.*)'\s*\z/, '\1') # enclosing single quotation marks
+      s.gsub!(/(_x005f_|_)x000d_/i, "\n") # fix sometimes encoding for newline
+      s.gsub!("\u00ad", "")               # strip soft hyphen
+      s.gsub!("\u0092", "'")              # windows-1252 apostrophe - https://stackoverflow.com/a/15564279/2866660
+      s.gsub!("‚", ",")                   # normalize unicode comma
+      s.gsub!("aÄs", "aïs")               # encoding issue for maïs
+      s.gsub!("ï", "ï")                  # encoding issue
+      s.gsub!("ë", "ë")                  # encoding issue
+      s.gsub!(/\A\s*(["']+)(.*)\1\s*\z/, '\2') # enclosing quotation marks
       s
     end
 

data/lib/food_ingredient_parser/loose/scanner.rb

@@ -4,7 +4,7 @@ module FoodIngredientParser::Loose
   class Scanner
 
     SEP_CHARS  = "|;,.".freeze
-    MARK_CHARS = "¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº⁽⁾†‡•°▪◊#^*".freeze
+    MARK_CHARS = "¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº⁽⁾†‡⁺•°▪◊#^˄*~".freeze
     PREFIX_RE  = /\A\s*(ingredients|contains|ingred[iï][eë]nt(en)?(declaratie)?|bevat|dit zit er\s?in|samenstelling|zutaten)\b\s*[:;.]?\s*/i.freeze
     NOTE_RE    = /\A\b(dit product kan\b|deze verpakking kan\b|kan sporen\b.*?\bbevatten\b|voor allergenen\b|allergenen\b|allergie[- ]informatie(\s*:|\b)|E\s*=|gemaakt in\b|geproduceerd in\b|bevat mogelijk\b|kijk voor meer\b|allergie-info|in de fabriek\b|in dit bedrijf\b|voor [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bis [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bgebruikt\b)/i.freeze
     # Keep in sync with +abbrev+ in the +Common+ grammar, plus relevant ones from the +Amount+ grammar.
@@ -17,8 +17,9 @@ module FoodIngredientParser::Loose
         L\.\s+rhamnosus\b | L\.\s+acidophilus\b | L\.\s+casei\b | B\.\s+lactis | A\.\s+oryzae |
         S\.\s+thermophilus\b | L\.\sbulgaricus\b |
         T\.\s*aestivum\b(\s+vitt\.)? |
+        nucifera\s+L\. |
         type\s+"\d+" |
-        E-e?\d{3}[a-z]?\s*\(i+\) |
+        E(-|\s+)?\d{3}[a-z]?\s*(\([iv]+\)|\[[iv]+\]) |
         www\.[-_\/:%.A-Za-z0-9]+
       )/xi,
       *%w[

data/lib/food_ingredient_parser/loose/transform/split_e_numbers.rb

@@ -7,8 +7,8 @@ module FoodIngredientParser::Loose
       #
       # @note mark and amount is lost, this is not expected on e-numbers
 
-      SPLIT_RE  = /\s*-\s*/.freeze
-      SINGLE_RE = /E-?\d{3}[a-z]?(?:\s*\(i+\))?/i.freeze
+      SPLIT_RE  = /\s*(-|\ben\b|\band\b|\bund\b|\bet\b)\s*/.freeze
+      SINGLE_RE = /E(-|\s+)?\d{3}[a-z]?\s*(\([iv]+\)|\[[iv]+\])?/i.freeze
       MATCH_RE  = /\A\s*(#{SINGLE_RE})(?:#{SPLIT_RE}(#{SINGLE_RE}))+\s*\z/i.freeze
 
       def self.transform!(node)

data/lib/food_ingredient_parser/strict/grammar/amount.treetop

@@ -21,22 +21,28 @@ module FoodIngredientParser::Strict::Grammar
       ) ws* )?
       amount_simple_quantity
       ( ws+ (
-        'of'i / 'or less of'i / 'or more of'i /
-        'minimaal'i / 'minimum'i / 'van het uitlekgewicht'i / 'van het geheel'i /
+        'of a'i / 'of'i / 'or less of'i / 'or more of'i /
+        'van een'i / 'minimaal'i / 'minimum'i / 'van het uitlekgewicht'i / 'van het geheel'i /
         'min.'i / 'min'i / 'max.'i / 'max'i
       ) )?
     end
 
     rule amount_simple_quantity
-      amount_simple_number ( ws* amount_simple_unit? ws* dash ws* amount_simple_number )? ws* amount_simple_unit
+      amount_simple_number ( ws* amount_simple_unit? ws* dash ws* amount_simple_number )? ( ws* amount_simple_unit )?
     end
 
     rule amount_simple_number
-      ( [±∓~∼∽≂≃≈≲≤<>≥≳] ws* )? number
+      ( amount_simple_comparator ws* )? number
+    end
+
+    rule amount_simple_comparator
+      '=' ws* [<>] /
+      [<>] ws* ( '=' / 'of gelijk aan'i !char / 'or equal to'i !char ) /
+      [±∓~∼∽≂≃≈≲≤<>≥≳] / '+/-' / '-/+'
     end
 
     rule amount_simple_unit
-      ( percent / ( ( 'procent' / 'percent' / 'gram'i / 'ml'i / 'mg'i / 'gr'i / 'g'i ) !char ) )
+      ( percent / ( ( 'procent' / 'percent' / 'gram'i / 'ml'i / 'mg'i / 'gr'i / 'g'i / 'ppm'i ) !char ) )
       ( ws 'vol'i ( !char / '.' ) )?
       ( ws* '℮' )?
     end

data/lib/food_ingredient_parser/strict/grammar/common.treetop

@@ -12,15 +12,20 @@ module FoodIngredientParser::Strict::Grammar
     rule char
       !mark [[:alnum:]] /
       fraction /
-      [-/\`'´‘’+=_{}&] /
-      [®™] /
-      [¿?] /                         # weird characters turning up in names (e.g. encoding issues)
+      [-/\`'"´‘’+=_{}&] /
+      [®©™♣] /
+      [¿?¯] /                        # weird characters turning up in names (e.g. encoding issues)
       [₁₂₃₄₅₆₇₈₉]                    # can occur with vitamins
     end
 
     rule mark
       # mark referencing a footnote
-      [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾'? / '⁽' [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾' / [†‡•°▪◊#^] / '*'+ / '(' ws* ( [†‡•°▪◊#^] / '*'+ ) ws* ')'
+      [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾'? /
+      '⁽' [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾' /
+      [˄^] digit /
+      [†‡⁺•°▪◊#˄^~˛] /
+      '*'+ /
+      '(' ws* ( [†‡⁺•°▪◊#˄^~˛] / '*'+ ) ws* ')'
     end
 
     rule digit
@@ -28,7 +33,8 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule fraction
-      [½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅐⅛⅜⅝⅞⅑⅒]
+      [½⅓⅔¼¾⅕⅖⅗⅘⅙⅚⅐⅛⅜⅝⅞⅑⅒] /
+      digit+ '/' digit+
     end
 
     rule percent
@@ -52,8 +58,22 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule e_number
-      ( 'E'i '-'? [0-9] [0-9] [0-9] [[:alpha:]]? )
-      ![[:alnum:]] / ( ws* '(' 'i'i+ ')' ) # e.g. "E450 (iii)"
+      ( 'E'i ( '-' / ws+ )? [0-9] [0-9] [0-9] [[:alpha:]]? )
+      ( ( ws* '(' [iIvV]+ ')' ) / ( ws* '[' [iIvV]+ ']' ) / ![[:alnum:]] ) # e.g. "E450 (iii)"
+    end
+
+    rule chem_systematic_name
+      ( chem_systematic_name_num dash ) ( chem_systematic_name_word dash chem_systematic_name_num dash ws? )* chem_systematic_name_word /
+      ( chem_systematic_name_word dash chem_systematic_name_num dash ws? )+ chem_systematic_name_word
+    end
+
+    rule chem_systematic_name_word
+      [A-Za-z]+ ( dash [A-Za-z]+ dash [A-Za-z]+ )*
+    end
+
+    rule chem_systematic_name_num
+      digit+ [RH] /
+      digit+ ( ',' digit+ )* '\''?
     end
 
     rule abbrev
@@ -110,6 +130,7 @@ module FoodIngredientParser::Strict::Grammar
         'w.o'i /
         'w.v'i /
         # not auto-generated additions
+        'nr.'i /
         'vit'i /   # vitamin
         'denat'i / # denaturated
         'alc'i /   # alcohol
@@ -131,8 +152,10 @@ module FoodIngredientParser::Strict::Grammar
         'L.' ws+ 'rhamnosus'i / 'L.' ws+ 'acidophilus'i / 'L.' ws+ 'casei' / 'B.'i ws+ 'lactis'i / 'A.'i ws+ 'oryzae'i /
         'S.' ws+ 'thermophilus'i / 'L.' ws+ 'bulgaricus'i /
         'T.' ws* 'aestivum'i (ws+ 'vitt.'i)? /
+        'nucifera' ws+ 'L.'i /
         'type'i ws+ '"' [0-9]+ '"' /
-        e_number
+        e_number /
+        chem_systematic_name
       ) ![[:alpha:]]
     end
   end

data/lib/food_ingredient_parser/strict/grammar/list_coloned.treetop

@@ -5,15 +5,15 @@ module FoodIngredientParser::Strict::Grammar
     include Ingredient
 
     rule list_coloned
-      contains:( ( ws* list_coloned_ingredient ws* '.,')+ ws* list_coloned_ingredient ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* '.,')+                             ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* '.' )+ ws* list_coloned_ingredient ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* '.' )+                             ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* ';,')+ ws* list_coloned_ingredient ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* ';,')+                             ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* ';' )+ ws* list_coloned_ingredient ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* ';' )+                             ) <ListNode> /
-      contains:(   ws* list_coloned_ingredient )                                        <ListNode>
+      contains:( ( ws* list_coloned_ingredient ws* '.' ws* ',' )+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* '.' ws* ',' )+                             ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* '.'         )+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* '.'         )+                             ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';' ws* ',' )+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';' ws* ',' )+                             ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';'         )+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';'         )+                             ) <ListNode> /
+      contains:(   ws* list_coloned_ingredient )                                                <ListNode>
     end
 
     rule list_coloned_inner_list
@@ -22,7 +22,7 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule list_coloned_ingredient
-      ing:ingredient_simple_with_amount ws* ':' ws* amount:amount post:( ws* '}' )?                    <IngredientNode> /
+      ing:ingredient_simple_with_amount ws* ':' ws* amount:amount post:( ws* '}' )? !( ws* word )      <IngredientNode> /
       ing:ingredient_simple_with_amount ws* ':' post:( ws* '}' )? ws* contains:list_coloned_inner_list <NestedIngredientNode>
     end
 

data/lib/food_ingredient_parser/strict/grammar/root.treetop

@@ -40,7 +40,7 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule root_mark_sentences_in_list
-      ( ( ws* [,.;] / ws ) ws* root_mark_sentence_in_list )+
+      ( ( ws* [,.;] / ws )+ root_mark_sentence_in_list )+
     end
 
     rule root_mark_sentence_in_list

data/lib/food_ingredient_parser/version.rb

@@ -1,4 +1,4 @@
 module FoodIngredientParser
-  VERSION      = '1.1.5'
-  VERSION_DATE = '2019-11-14'
+  VERSION      = '1.1.10'
+  VERSION_DATE = '2021-03-23'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: food_ingredient_parser
 version: !ruby/object:Gem::Version
-  version: 1.1.5
+  version: 1.1.10
 platform: ruby
 authors:
 - wvengen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-14 00:00:00.000000000 Z
+date: 2021-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: treetop
@@ -87,8 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Parser for ingredient lists found on food products.