food_ingredient_parser 1.1.9 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e83f2b90f83d4cb0deb193140dcce6f4ee1700a80c781c95b7fa09219da9f69
-  data.tar.gz: '0929b958e93cf8e61e54b5e268ad32135aa45e87d1c163ae912790de55a85e28'
+  metadata.gz: 7c478a080e36c8f48ee3dbd6e9978eadec3758a4b0ab6fab571e18f103ed6bf0
+  data.tar.gz: aa078366f72ab03d038d497c908a3ad92f5816f37d3f0308fa64e81680905dea
 SHA512:
-  metadata.gz: 929fd7057be9e1f35ec1db55e08bad1078d8bc1f977b07b25c26f3f0ce0919e0cb82ca4379e050067b20ab5cee760fb10c2d267d60fc7555d18af1e8ae9ca9e9
-  data.tar.gz: 4873942e33ad823fbf84391cc70c4f4760cf0d93cceced307814d3abf73f5e91af8529fcfa048345b0a896dd967deac729d9051ab7e89b3c4174c8ee42f9f61b
+  metadata.gz: d8acbd71e431958a72350e6fd1d3e5e8d21db8ee53525c53a08bbe2c564734fca9601ac0fdc33d9737695f292bc7cd6da898721f02f68ca8f87175c5b276c709
+  data.tar.gz: f261a1537a6e903d55b36dc91c0a1c302893d7a092a83afb8598730c87142d041fdaf86540918697f297f5024461082e5e56594df64f603f56f4b22148c7c9fd

data/README.md

@@ -69,6 +69,7 @@ Usage: bin/food_ingredient_parser [options] --file|-f <filename>
     -r, --parser PARSER              Use specific parser (strict, loose).
     -e, --[no-]escape                Escape newlines
     -c, --[no-]color                 Use color
+        --[no-]html                  Print as HTML with parsing markup
     -v, --[no-]verbose               Show more data (parsed tree).
         --version                    Show program version.
     -h, --help                       Show this help
@@ -103,6 +104,9 @@ RootNode+Root3 offset=0, "tomato" (contains,notes):
   SyntaxNode offset=6, ""
 {:contains=>[{:name=>"tomato"}]}
 
+$ food_ingredient_parser --html -s "tomato"
+<div class="root"><span class='depth0'><span class='name'>tomato</span></span></div>
+
 $ food_ingredient_parser -v -r loose -s "tomato"
 "tomato"
 Node interval=0..5
@@ -193,7 +197,7 @@ plus a bit of English and German. Support for other languages is already good, b
 areas: improvements are welcome (starting with a corpus in [data/](data/)).
 
 Many ingredient lists from the USA are structured a bit differently than those from Europe, they
-parse less well (that that's a matter of tine-tuning).
+parse less well (that is probably a matter of tine-tuning).
 
 ## Test data
 

data/bin/food_ingredient_parser

@@ -2,6 +2,7 @@
 #
 # Parser for food ingredient lists.
 #
+require 'cgi'
 require 'optparse'
 
 $:.push(File.expand_path(File.dirname(__FILE__) + "/../lib"))
@@ -31,24 +32,31 @@ def colorize(color, s)
   end
 end
 
-def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false, color: false)
+def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false, color: false, html: false)
   parsed ||= parser.parse(s)
 
   return unless print.nil? || (parsed && print == :parsed) || (!parsed && print == :noresult)
 
-  puts colorize(color && "0;32", escape ? s.gsub("\n", "\\n") : s) if verbosity > 0
+  puts colorize(color && "0;32", escape ? s.gsub("\n", "\\n") : s) if !html && verbosity > 0
 
-  if parsed
+  if !html && parsed
     puts(parsed.inspect) if verbosity > 1
     pp(parsed.to_h, color: color) if verbosity > 0
-    return true
-  else
+
+  elsif !html && !parsed
     puts "(no result: #{parser.parser.failure_reason})" if verbosity > 0
-    return false
+
+  elsif html && parsed
+    puts('<div class="root">' + parsed.to_html + '</div>') if verbosity > 0
+
+  else
+    puts('<div class="root">' + CGI.escapeHTML(parsed) + '</div>') if verbosity > 0
   end
+
+  return !!parsed
 end
 
-def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: false)
+def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: false, html: false)
   count_parsed = count_noresult = 0
   File.foreach(path) do |line|
     next if line =~ /^#/    # comment
@@ -59,7 +67,7 @@ def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: fa
     count_parsed += 1 if parsed
     count_noresult += 1 unless parsed
 
-    parse_single(line, parsed, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color)
+    parse_single(line, parsed, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color, html: html)
   end
 
   pct_parsed   = 100.0 * count_parsed / (count_parsed + count_noresult)
@@ -75,6 +83,7 @@ print = nil
 parser_name = :strict
 escape = false
 color = true
+html = false
 PARSERS = {
   strict: FoodIngredientParser::Strict::Parser,
   loose:  FoodIngredientParser::Loose::Parser
@@ -95,6 +104,7 @@ OptionParser.new do |opts|
   opts.on("-r", "--parser PARSER", "Use specific parser (#{PARSERS.keys.join(", ")}).") {|p| parser_name = p&.downcase&.to_sym }
   opts.on("-e", "--[no-]escape", "Escape newlines") {|e| escape = !!e }
   opts.on("-c", "--[no-]color", "Use color") {|e| color = !!e }
+  opts.on(      "--[no-]html", "Print as HTML with parsing markup") {|e| html = !!e }
   opts.on("-v", "--[no-]verbose", "Show more data (parsed tree).") {|v| verbosity = v ? 2 : 1 }
   opts.on(      "--version", "Show program version.") do
     puts("food_ingredient_parser v#{FoodIngredientParser::VERSION}")
@@ -112,8 +122,8 @@ if strings.any? || files.any?
     exit(1)
   end
   success = true
-  strings.each {|s| success &= parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
-  files.each   {|f| success &= parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) == 0 }
+  strings.each {|s| success &= parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color, html: html) }
+  files.each   {|f| success &= parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color, html: html) == 0 }
   success or exit(1)
 else
   STDERR.puts("Please specify one or more --file or --string arguments (see --help).")

data/lib/food_ingredient_parser/loose/scanner.rb

@@ -4,8 +4,9 @@ module FoodIngredientParser::Loose
   class Scanner
 
     SEP_CHARS  = "|;,.".freeze
+    AND_SEP_RE = /\A\s*(and|en|und)\s+/i.freeze
     MARK_CHARS = "¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº⁽⁾†‡⁺•°▪◊#^˄*~".freeze
-    PREFIX_RE  = /\A\s*(ingredients|contains|ingred[iï][eë]nt(en)?(declaratie)?|bevat|dit zit er\s?in|samenstelling|zutaten)\b\s*[:;.]?\s*/i.freeze
+    PREFIX_RE  = /\A\s*(ingredients(\s*list)?|contains|ingred[iï][eë]nt(en)?(declaratie)?|bevat|dit zit er\s?in|samenstelling|zutaten)\b\s*[:;.]?\s*/i.freeze
     NOTE_RE    = /\A\b(dit product kan\b|deze verpakking kan\b|kan sporen\b.*?\bbevatten\b|voor allergenen\b|allergenen\b|allergie[- ]informatie(\s*:|\b)|E\s*=|gemaakt in\b|geproduceerd in\b|bevat mogelijk\b|kijk voor meer\b|allergie-info|in de fabriek\b|in dit bedrijf\b|voor [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bis [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bgebruikt\b)/i.freeze
     # Keep in sync with +abbrev+ in the +Common+ grammar, plus relevant ones from the +Amount+ grammar.
     ABBREV_RE  = Regexp.union(
@@ -19,12 +20,12 @@ module FoodIngredientParser::Loose
         T\.\s*aestivum\b(\s+vitt\.)? |
         nucifera\s+L\. |
         type\s+"\d+" |
-        E[- ]?\d{3}[a-z]?\s*\([iv]+\) |
+        E(-|\s+)?\d{3}[a-z]?\s*(\([iv]+\)|\[[iv]+\]) |
         www\.[-_\/:%.A-Za-z0-9]+
       )/xi,
       *%w[
-        a.o.p b.g.a b.o.b c.a c.i d.e d.m.v d.o.c d.o.p d.s e.a e.g e.u f.i.l f.o.s i.a
-        i.d i.e i.g.m.e i.g.p i.m.v i.o i.v.m l.s.l n.a n.b n.o n.v.t o.a o.b.v p.d.o
+        a.o.p b.g.a b.o.b c.a c.i d.e d.m.v d.o.c d.o.p d.s e.a e.g e.u f.i.l f.o.s h.o.h
+        i.a i.d i.e i.g.m.e i.g.p i.m.v i.o i.v.m l.s.l n.a n.b n.o n.v.t o.a o.b.v p.d.o
         p.g.i q.s s.l s.s t.o.v u.h.t v.g v.s w.a w.o w.v vit denat alc vol conc subsp
         min max ca
       ].map {|s| /\A#{Regexp.escape(s)}\b\.?/i}
@@ -75,6 +76,11 @@ module FoodIngredientParser::Loose
       elsif ")]".include?(c)      # close nesting
         add_child
         close_parent
+        # after bracket check for 'and' to not lose text
+        if is_and_sep?(@i+1)
+          @i += and_sep_len(@i+1)
+          add_child
+        end
       elsif is_notes_start?       # usually a dot marks the start of notes
         close_all_ancestors
         @iterator = :notes
@@ -148,6 +154,15 @@ module FoodIngredientParser::Loose
       chars.include?(c) && @s[@i-1..@i+1] !~ /\A\d.\d\z/
     end
 
+    def is_and_sep?(i = @i)
+      and_sep_len(i) > 0
+    end
+
+    def and_sep_len(i = @i)
+      m = @s[i..-1].match(AND_SEP_RE)
+      m ? m.offset(0).last : 0
+    end
+
     def is_mark?(i = @i)
       mark_len(i) > 0 && @s[i..i+1] !~ /\A°[CF]/
     end

data/lib/food_ingredient_parser/loose/transform/split_e_numbers.rb

@@ -7,8 +7,8 @@ module FoodIngredientParser::Loose
       #
       # @note mark and amount is lost, this is not expected on e-numbers
 
-      SPLIT_RE  = /\s*-\s*/.freeze
-      SINGLE_RE = /E-?\d{3}[a-z]?(?:\s*\([iv]+\))?/i.freeze
+      SPLIT_RE  = /\s*(-|\ben\b|\band\b|\bund\b|\bet\b)\s*/.freeze
+      SINGLE_RE = /E(-|\s+)?\d{3}[a-z]?\s*(\([iv]+\)|\[[iv]+\])?/i.freeze
       MATCH_RE  = /\A\s*(#{SINGLE_RE})(?:#{SPLIT_RE}(#{SINGLE_RE}))+\s*\z/i.freeze
 
       def self.transform!(node)

data/lib/food_ingredient_parser/strict/grammar/common.treetop

@@ -58,8 +58,8 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule e_number
-      ( 'E'i '-'? [0-9] [0-9] [0-9] [[:alpha:]]? )
-      ( ( ws* '(' [iIvV]+ ')' ) / ![[:alnum:]] ) # e.g. "E450 (iii)"
+      ( 'E'i ( '-' / ws+ )? [0-9] [0-9] [0-9] [[:alpha:]]? )
+      ( ( ws* '(' [iIvV]+ ')' ) / ( ws* '[' [iIvV]+ ']' ) / ![[:alnum:]] ) # e.g. "E450 (iii)"
     end
 
     rule chem_systematic_name
@@ -102,6 +102,7 @@ module FoodIngredientParser::Strict::Grammar
         'e.u'i /
         'f.i.l'i /
         'f.o.s'i /
+        'h.o.h'i /
         'i.a'i /
         'i.d'i /
         'i.e'i /

data/lib/food_ingredient_parser/strict/grammar/ingredient.treetop

@@ -5,7 +5,12 @@ module FoodIngredientParser::Strict::Grammar
     include IngredientColoned
 
     rule ingredient
-      ws* ( ingredient_nested / ingredient_coloned / ingredient_simple_with_amount )
+      ws*
+      (
+        ingredient_nested ( ws* and ws+ ingredient )? /
+        ingredient_coloned /
+        ingredient_simple_with_amount
+      )
     end
 
   end

data/lib/food_ingredient_parser/strict/grammar/list_coloned.treetop

@@ -5,15 +5,15 @@ module FoodIngredientParser::Strict::Grammar
     include Ingredient
 
     rule list_coloned
-      contains:( ( ws* list_coloned_ingredient ws* '.,')+ ws* list_coloned_ingredient ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* '.,')+                             ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* '.' )+ ws* list_coloned_ingredient ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* '.' )+                             ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* ';,')+ ws* list_coloned_ingredient ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* ';,')+                             ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* ';' )+ ws* list_coloned_ingredient ) <ListNode> /
-      contains:( ( ws* list_coloned_ingredient ws* ';' )+                             ) <ListNode> /
-      contains:(   ws* list_coloned_ingredient )                                        <ListNode>
+      contains:( ( ws* list_coloned_ingredient ws* '.' ws* ',' )+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* '.' ws* ',' )+                             ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* '.'         )+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* '.'         )+                             ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';' ws* ',' )+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';' ws* ',' )+                             ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';'         )+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';'         )+                             ) <ListNode> /
+      contains:(   ws* list_coloned_ingredient )                                                <ListNode>
     end
 
     rule list_coloned_inner_list
@@ -22,7 +22,7 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule list_coloned_ingredient
-      ing:ingredient_simple_with_amount ws* ':' ws* amount:amount post:( ws* '}' )?                    <IngredientNode> /
+      ing:ingredient_simple_with_amount ws* ':' ws* amount:amount post:( ws* '}' )? !( ws* word )      <IngredientNode> /
       ing:ingredient_simple_with_amount ws* ':' post:( ws* '}' )? ws* contains:list_coloned_inner_list <NestedIngredientNode>
     end
 

data/lib/food_ingredient_parser/strict/grammar/root.treetop

@@ -19,9 +19,10 @@ module FoodIngredientParser::Strict::Grammar
 
     rule root_prefix
       (
-        'ingredients'i / 'contains'i /
+        'ingredients'i ( ws+ 'list'i )? / 'contains'i /
         ('ingred'i [IÏiï] [EËeë] 'n'i ( 't'i 'en'i? 'declaratie'i? )? ) / 'bevat'i / 'dit zit er in'i / 'samenstelling'i /
-        'zutaten'i
+        'zutaten'i /
+        'ingredienser'i
       )
       ( ws* [:;.] ( ws* newline )? / ws* newline / ws ) ws*  # optional colon or other separator
       "'"? ws*                                               # stray quote occurs sometimes

data/lib/food_ingredient_parser/version.rb

@@ -1,4 +1,4 @@
 module FoodIngredientParser
-  VERSION      = '1.1.9'
-  VERSION_DATE = '2021-01-12'
+  VERSION      = '1.2.0'
+  VERSION_DATE = '2024-01-19'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: food_ingredient_parser
 version: !ruby/object:Gem::Version
-  version: 1.1.9
+  version: 1.2.0
 platform: ruby
 authors:
 - wvengen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-12 00:00:00.000000000 Z
+date: 2024-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: treetop
@@ -87,7 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Parser for ingredient lists found on food products.