food_ingredient_parser 1.1.1 → 1.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 41da81db767bedd583db2794f44d39d710ddef58
-  data.tar.gz: 9bc24f8e83d04fb608e8fba4364aa26117fd898b
+SHA256:
+  metadata.gz: e65fb234a122c893fcb4de453e4cac07851d5000b848fa99d2b356002e187a60
+  data.tar.gz: c72d35c6fc978a6ddc139e3adf2059c24104221627f04857ba73aad159b29a0f
 SHA512:
-  metadata.gz: 7634ebb4a8f3530cab515c5b27d2782df8c2521eb79df67ecbe08945360bcc27d68456d389d69a83808792145ecfb7094b3b97b5c33cbb17119642449813df88
-  data.tar.gz: 3b5c5ed78b7c4e41211855523acaf2a4c519b0e021a32eb698a715f7dccf2beacde229593d8422b71ce7a5a55dc1e266347d263c6629a72cdb308556697f7d93
+  metadata.gz: 295728820e893f1277bb06cfdad572df1fde9d496df230caf55f41b23d124677131b004a5ca0b2a2e4a25b2273f98feb18840c47c6361ec474082d66d4fde7fb
+  data.tar.gz: 0ca48100273055b6c8f6e1d89bd1b44976e325cc9633080d6570e52cb6ef5d07dd419013d9b9aff67f95b6e4fac6ae12396c9248123daf4902d3eb97d84d1b28

data/README.md

@@ -65,10 +65,10 @@ Usage: bin/food_ingredient_parser [options] --file|-f <filename>
     -s, --string INGREDIENTS         Parse specified ingredient list.
     -q, --[no-]quiet                 Only show summary.
     -p, --parsed                     Only show lines that were successfully parsed.
+    -n, --noresult                   Only show lines that had no result.
     -r, --parser PARSER              Use specific parser (strict, loose).
     -e, --[no-]escape                Escape newlines
     -c, --[no-]color                 Use color
-    -n, --noresult                   Only show lines that had no result.
     -v, --[no-]verbose               Show more data (parsed tree).
         --version                    Show program version.
     -h, --help                       Show this help
@@ -185,10 +185,23 @@ So if you only use the stable interface (`parse`, `to_h` and `to_html`), you can
 to e.g. `~> 1.0`. If you depend on more, lock your version against e.g. `~> 1.0.0` and test when you
 upgrade to `1.1`.
 
+## Languages
+
+While most of the parsing is language-independent, some parts need knowledge about certain words
+(like abbreviations and amount specifiers). The gem was developed with ingredient lists in Dutch (nl),
+plus a bit of English and German. Support for other languages is already good, but lacks in certain
+areas: improvements are welcome (starting with a corpus in [data/](data/)).
+
+Many ingredient lists from the USA are structured a bit differently than those from Europe, they
+parse less well (that that's a matter of tine-tuning).
+
 ## Test data
 
-[`data/ingredient-samples-nl`](data/ingredient-samples-nl) contains about 150k
+[`data/ingredient-samples-qm-nl`](data/ingredient-samples-qm-nl) contains about 150k
 real-world ingredient lists found on the Dutch market. Each line contains one ingredient
 list (newlines are encoded as `\n`, empty lines and those starting with `#` are ignored).
-The strict parser currently parses about three quarter, while the loose parser returns
-something for all of them.
+The strict parser currently parses 80%, while the loose parser returns something for all of them.
+
+## License
+
+This software is distributed under the [MIT license](LICENSE). Data may have a [different license](data/README.md).

data/bin/food_ingredient_parser

@@ -41,8 +41,10 @@ def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false
   if parsed
     puts(parsed.inspect) if verbosity > 1
     pp(parsed.to_h, color: color) if verbosity > 0
+    return true
   else
     puts "(no result: #{parser.parser.failure_reason})" if verbosity > 0
+    return false
   end
 end
 
@@ -63,6 +65,7 @@ def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: fa
   pct_parsed   = 100.0 * count_parsed / (count_parsed + count_noresult)
   pct_noresult = 100.0 * count_noresult / (count_parsed + count_noresult)
   puts "parsed #{colorize(color && "1;32", count_parsed)} (#{pct_parsed.round(1)}%), no result #{colorize(color && "1;31", count_noresult)} (#{pct_noresult.round(1)}%)"
+  return count_noresult
 end
 
 verbosity = 1
@@ -88,10 +91,10 @@ OptionParser.new do |opts|
 
   opts.on("-q", "--[no-]quiet", "Only show summary.") {|q| verbosity = q ? 0 : 1 }
   opts.on("-p", "--parsed", "Only show lines that were successfully parsed.") {|p| print = :parsed }
+  opts.on("-n", "--noresult", "Only show lines that had no result.") {|p| print = :noresult }
   opts.on("-r", "--parser PARSER", "Use specific parser (#{PARSERS.keys.join(", ")}).") {|p| parser_name = p&.downcase&.to_sym }
   opts.on("-e", "--[no-]escape", "Escape newlines") {|e| escape = !!e }
   opts.on("-c", "--[no-]color", "Use color") {|e| color = !!e }
-  opts.on("-n", "--noresult", "Only show lines that had no result.") {|p| print = :noresult }
   opts.on("-v", "--[no-]verbose", "Show more data (parsed tree).") {|v| verbosity = v ? 2 : 1 }
   opts.on(      "--version", "Show program version.") do
     puts("food_ingredient_parser v#{FoodIngredientParser::VERSION}")
@@ -108,8 +111,10 @@ if strings.any? || files.any?
     STDERR.puts("Please specify one of the known parsers: #{PARSERS.keys.join(", ")}.")
     exit(1)
   end
-  strings.each {|s| parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
-  files.each   {|f| parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
+  success = true
+  strings.each {|s| success &= parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
+  files.each   {|f| success &= parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) == 0 }
+  success or exit(1)
 else
   STDERR.puts("Please specify one or more --file or --string arguments (see --help).")
 end

data/lib/food_ingredient_parser/cleaner.rb

@@ -4,6 +4,7 @@ module FoodIngredientParser
     def self.clean(s)
       s.gsub!("\u00ad", "")             # strip soft hyphen
       s.gsub!("\u0092", "'")            # windows-1252 apostrophe - https://stackoverflow.com/a/15564279/2866660
+      s.gsub!("‚", ",")                 # normalize unicode comma
       s.gsub!("aÄs", "aïs")             # encoding issue for maïs
       s.gsub!("ï", "ï")                # encoding issue
       s.gsub!("ë", "ë")                # encoding issue

data/lib/food_ingredient_parser/loose/parser.rb

@@ -2,6 +2,7 @@ require_relative '../cleaner'
 require_relative 'scanner'
 require_relative 'transform/amount'
 require_relative 'transform/handle_missing_name'
+require_relative 'transform/split_e_numbers'
 
 module FoodIngredientParser::Loose
   class Parser
@@ -20,6 +21,7 @@ module FoodIngredientParser::Loose
       s = FoodIngredientParser::Cleaner.clean(s) if clean
       n = Scanner.new(s).scan
       n = Transform::Amount.transform!(n) if n
+      n = Transform::SplitENumbers.transform!(n) if n
       n = Transform::HandleMissingName.transform!(n) if n && normalize
       n
     end

data/lib/food_ingredient_parser/loose/scanner.rb

@@ -4,16 +4,31 @@ module FoodIngredientParser::Loose
   class Scanner
 
     SEP_CHARS  = "|;,.".freeze
-    MARK_CHARS = "¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº⁽⁾†‡•°#^*".freeze
+    MARK_CHARS = "¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº⁽⁾†‡•°▪◊#^˄*~".freeze
     PREFIX_RE  = /\A\s*(ingredients|contains|ingred[iï][eë]nt(en)?(declaratie)?|bevat|dit zit er\s?in|samenstelling|zutaten)\b\s*[:;.]?\s*/i.freeze
-    NOTE_RE    = /\A\b(dit product kan\b|deze verpakking kan\b|kan sporen\b.*?\bbevatten\b|voor allergenen\b|allergenen\b|E\s*=|gemaakt in\b|geproduceerd in\b|bevat mogelijk\b|kijk voor meer\b|allergie-info|in de fabriek\b|in dit bedrijf\b|voor [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bis [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bgebruikt\b)/i.freeze
+    NOTE_RE    = /\A\b(dit product kan\b|deze verpakking kan\b|kan sporen\b.*?\bbevatten\b|voor allergenen\b|allergenen\b|allergie[- ]informatie(\s*:|\b)|E\s*=|gemaakt in\b|geproduceerd in\b|bevat mogelijk\b|kijk voor meer\b|allergie-info|in de fabriek\b|in dit bedrijf\b|voor [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bis [0-9,.]+ (g\.?|gr\.?|ram|ml).*\bgebruikt\b)/i.freeze
     # Keep in sync with +abbrev+ in the +Common+ grammar, plus relevant ones from the +Amount+ grammar.
-    ABBREV_RE  = Regexp.union(/\A(N°|°C|(ijzer|chroom|koper)\s*\(I+\)\s*[[:alnum:]]+)\b/i, *%w[
-      a.o.p b.g.a b.o.b c.a c.i d.e d.m.v d.o.c d.o.p d.s e.a e.g e.u f.i.l f.o.s i.a
-      i.d i.e i.g.m.e i.g.p i.m.v i.o i.v.m l.s.l n.a n.b n.o n.v.t o.a o.b.v p.d.o
-      p.g.i q.s s.l s.s t.o.v u.h.t v.g v.s w.a w.o w.v vit denat
-      min max ca
-    ].map {|s| /\A#{Regexp.escape(s)}\b\.?/i}).freeze
+    ABBREV_RE  = Regexp.union(
+      /\A(
+        N°\b |
+        °C\b |
+        (ijzer|chroom|koper)\s*\(I+\)\s*[[:alnum:]]+\b |
+        L\(\+\)[ -][[:alnum:]]+\b |
+        L\.\s+rhamnosus\b | L\.\s+acidophilus\b | L\.\s+casei\b | B\.\s+lactis | A\.\s+oryzae |
+        S\.\s+thermophilus\b | L\.\sbulgaricus\b |
+        T\.\s*aestivum\b(\s+vitt\.)? |
+        nucifera\s+L\. |
+        type\s+"\d+" |
+        E-e?\d{3}[a-z]?\s*\(i+\) |
+        www\.[-_\/:%.A-Za-z0-9]+
+      )/xi,
+      *%w[
+        a.o.p b.g.a b.o.b c.a c.i d.e d.m.v d.o.c d.o.p d.s e.a e.g e.u f.i.l f.o.s i.a
+        i.d i.e i.g.m.e i.g.p i.m.v i.o i.v.m l.s.l n.a n.b n.o n.v.t o.a o.b.v p.d.o
+        p.g.i q.s s.l s.s t.o.v u.h.t v.g v.s w.a w.o w.v vit denat alc vol conc subsp
+        min max ca
+      ].map {|s| /\A#{Regexp.escape(s)}\b\.?/i}
+    ).freeze
 
     def initialize(s, index: 0)
       @s = s                           # input string

data/lib/food_ingredient_parser/loose/transform/amount_from_name.treetop

@@ -5,8 +5,9 @@ module FoodIngredientParser::Loose::Transform
 
     rule amount_from_name
       # just amount, amount in front or at the end
-      ws* amount:amount ws+ name:(.*) /
-      ws* amount:amount ws* /
+      ws* amount:amount                ws+ name:(.*) /
+      ws* amount:amount_simple_percent ws* name:(.*) /
+      ws* amount:amount                ws* /
       ws* name:( !amount word ( ws+ !amount word )* )+ ws* amount:amount ws*
     end
   end

data/lib/food_ingredient_parser/loose/transform/split_e_numbers.rb

@@ -0,0 +1,51 @@
+require_relative '../node'
+
+module FoodIngredientParser::Loose
+  module Transform
+    class SplitENumbers
+      # Transforms node tree to split e-number combinations.
+      #
+      # @note mark and amount is lost, this is not expected on e-numbers
+
+      SPLIT_RE  = /\s*-\s*/.freeze
+      SINGLE_RE = /E-?\d{3}[a-z]?(?:\s*\(i+\))?/i.freeze
+      MATCH_RE  = /\A\s*(#{SINGLE_RE})(?:#{SPLIT_RE}(#{SINGLE_RE}))+\s*\z/i.freeze
+
+      def self.transform!(node)
+        new(node).transform!
+      end
+
+      def initialize(node)
+        @node = node
+      end
+
+      def transform!
+        transform_node!(@node)
+        @node
+      end
+
+      private
+
+      def transform_node!(node)
+        if node.contains.any?
+          node.contains.each {|n| transform_node!(n) }
+        elsif node.name && m = MATCH_RE.match(node.name.text_value)
+          i = 0
+          while m = node.name.text_value.match(SPLIT_RE, i)
+            node.contains << new_node(node, i, m.begin(0)-1)
+            i = m.end(0)
+          end
+          node.contains << new_node(node, i, node.name.interval.last) if i <= node.name.interval.last
+          node.name = nil
+        end
+      end
+
+      def new_node(node, begins, ends)
+        offset = node.name.interval.first
+        new_node = Node.new(node.input, offset + begins .. offset + ends)
+        new_node.name = Node.new(node.input, new_node.interval)
+        new_node
+      end
+    end
+  end
+end

data/lib/food_ingredient_parser/strict/grammar/amount.treetop

@@ -9,6 +9,10 @@ module FoodIngredientParser::Strict::Grammar
       amount:amount_simple                 <AmountNode>
     end
 
+    rule amount_simple_percent
+      amount:(amount_simple_number ws* percent) <AmountNode>
+    end
+
     rule amount_simple
       ( (
         'of which'i / 'at least'i / 'minimal'i / 'maximal'i / 'less than'i / 'more than'i /
@@ -17,6 +21,7 @@ module FoodIngredientParser::Strict::Grammar
       ) ws* )?
       amount_simple_quantity
       ( ws+ (
+        'of'i / 'or less of'i / 'or more of'i /
         'minimaal'i / 'minimum'i / 'van het uitlekgewicht'i / 'van het geheel'i /
         'min.'i / 'min'i / 'max.'i / 'max'i
       ) )?
@@ -32,6 +37,8 @@ module FoodIngredientParser::Strict::Grammar
 
     rule amount_simple_unit
       ( percent / ( ( 'procent' / 'percent' / 'gram'i / 'ml'i / 'mg'i / 'gr'i / 'g'i ) !char ) )
+      ( ws 'vol'i ( !char / '.' ) )?
+      ( ws* '℮' )?
     end
   end
 end

data/lib/food_ingredient_parser/strict/grammar/common.treetop

@@ -10,17 +10,22 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule char
-      [[:alnum:]] /
+      !mark [[:alnum:]] /
       fraction /
-      [-/\`'´’+=_{}&] /
-      [®™] /
-      [¿?] /                         # weird characters turning up in names (e.g. encoding issues)
+      [-/\`'"´‘’+=_{}&] /
+      [®©™♣] /
+      [¿?¯] /                        # weird characters turning up in names (e.g. encoding issues)
       [₁₂₃₄₅₆₇₈₉]                    # can occur with vitamins
     end
 
     rule mark
       # mark referencing a footnote
-      [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾'? / '⁽' [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾' / [†‡•°#^] / '*'+ / '(' ws* ( [†‡•°#^] / '*'+ ) ws* ')'
+      [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾'? /
+      '⁽' [¹²³⁴⁵ᵃᵇᶜᵈᵉᶠᵍªº] '⁾' /
+      [˄^] digit /
+      [†‡•°▪◊#˄^~˛] /
+      '*'+ /
+      '(' ws* ( [†‡•°▪◊#˄^~˛] / '*'+ ) ws* ')'
     end
 
     rule digit
@@ -48,7 +53,12 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule and
-      ( 'and' / 'en' / 'und' / '&' ) !char
+      ( 'and' / 'en' / 'und' ) !char / '&'
+    end
+
+    rule e_number
+      ( 'E'i '-'? [0-9] [0-9] [0-9] [[:alpha:]]? )
+      ![[:alnum:]] / ( ws* '(' 'i'i+ ')' ) # e.g. "E450 (iii)"
     end
 
     rule abbrev
@@ -105,8 +115,13 @@ module FoodIngredientParser::Strict::Grammar
         'w.o'i /
         'w.v'i /
         # not auto-generated additions
-        'vit'i /
-        'denat'i
+        'vit'i /   # vitamin
+        'denat'i / # denaturated
+        'alc'i /   # alcohol
+        'vol'i /   # volume
+        'conc'i /  # concentration
+        'subsp'i / # subspecies
+        'www.'i [-_\/:%.A-Za-z0-9]+
       )
       '.'? ![[:alpha:]]
     end
@@ -116,7 +131,14 @@ module FoodIngredientParser::Strict::Grammar
       (
         'N°'i /
         '°C'i /
-        ( 'ijzer'i / 'chroom'i / 'koper'i ) ws* '(' 'I'i+ ')' ws* [[:alnum:]]+
+        ( 'ijzer'i / 'chroom'i / 'koper'i ) ws* '(' 'I'i+ ')' ws* [[:alnum:]]+ /
+        'L(+)' ('-' / ws) [[:alnum:]]+ /
+        'L.' ws+ 'rhamnosus'i / 'L.' ws+ 'acidophilus'i / 'L.' ws+ 'casei' / 'B.'i ws+ 'lactis'i / 'A.'i ws+ 'oryzae'i /
+        'S.' ws+ 'thermophilus'i / 'L.' ws+ 'bulgaricus'i /
+        'T.' ws* 'aestivum'i (ws+ 'vitt.'i)? /
+        'nucifera' ws+ 'L.'i /
+        'type'i ws+ '"' [0-9]+ '"' /
+        e_number
       ) ![[:alpha:]]
     end
   end

data/lib/food_ingredient_parser/strict/grammar/ingredient_coloned.treetop

@@ -10,7 +10,8 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule ingredient_coloned_inner_list
-      contains:( ingredient_coloned_simple_with_amount_and_nest ws* ( '/'+ ws* ingredient_coloned_simple_with_amount_and_nest )* ) <ListNode>
+      contains:( ingredient_simple_e_number ( ws* dash ws* ingredient_simple_e_number )+ ) <ListNode> /
+      contains:( ingredient_coloned_simple_with_amount_and_nest ( ws* '/'+ ws* ingredient_coloned_simple_with_amount_and_nest )* ) <ListNode>
     end
 
     # @see IngredientSimple#ingredient_simple

data/lib/food_ingredient_parser/strict/grammar/ingredient_nested.treetop

@@ -21,7 +21,7 @@ module FoodIngredientParser::Strict::Grammar
 
     rule ingredient_nested_contains
       'contains'i /
-      'bevat'i
+      'bevat'i / 'bevat o.a.'i / 'o.a.'i / 'met'i
     end
 
   end

data/lib/food_ingredient_parser/strict/grammar/ingredient_simple.treetop

@@ -9,10 +9,15 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule ingredient_simple_with_amount
-      pre:( '{' ws* )? amount:amount ws+ ing:ingredient_simple                   <IngredientNode> /
+      pre:( '{' ws* )? amount:amount                ws+ ing:ingredient_simple    <IngredientNode> /
+      pre:( '{' ws* )? amount:amount_simple_percent ws* ing:ingredient_simple    <IngredientNode> /
       ing:ingredient_simple ws* amount:amount post:( ws* '}' )? (ws? mark:mark)? <IngredientNode> /
       ing:ingredient_simple                                                      <IngredientNode>
     end
 
+    rule ingredient_simple_e_number
+      name:e_number <IngredientNode>
+    end
+
   end
 end

data/lib/food_ingredient_parser/strict/grammar/list_coloned.treetop

@@ -5,14 +5,19 @@ module FoodIngredientParser::Strict::Grammar
     include Ingredient
 
     rule list_coloned
+      contains:( ( ws* list_coloned_ingredient ws* '.,')+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* '.,')+                             ) <ListNode> /
       contains:( ( ws* list_coloned_ingredient ws* '.' )+ ws* list_coloned_ingredient ) <ListNode> /
       contains:( ( ws* list_coloned_ingredient ws* '.' )+                             ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';,')+ ws* list_coloned_ingredient ) <ListNode> /
+      contains:( ( ws* list_coloned_ingredient ws* ';,')+                             ) <ListNode> /
       contains:( ( ws* list_coloned_ingredient ws* ';' )+ ws* list_coloned_ingredient ) <ListNode> /
       contains:( ( ws* list_coloned_ingredient ws* ';' )+                             ) <ListNode> /
       contains:(   ws* list_coloned_ingredient )                                        <ListNode>
     end
 
     rule list_coloned_inner_list
+      contains:( ingredient_simple_e_number ( ws* dash ws* ingredient_simple_e_number )+ ) <ListNode> /
       contains:( ingredient ( ws* ',' ws* ingredient )* ) <ListNode>
     end
 

data/lib/food_ingredient_parser/strict/grammar/root.treetop

@@ -40,7 +40,7 @@ module FoodIngredientParser::Strict::Grammar
     end
 
     rule root_mark_sentences_in_list
-      ( ( ws* [,.;] / ws ) ws* root_mark_sentence_in_list )+
+      ( ( ws* [,.;] / ws )+ root_mark_sentence_in_list )+
     end
 
     rule root_mark_sentence_in_list

data/lib/food_ingredient_parser/version.rb

@@ -1,4 +1,4 @@
 module FoodIngredientParser
-  VERSION      = '1.1.1'
-  VERSION_DATE = '2018-09-25'
+  VERSION      = '1.1.6'
+  VERSION_DATE = '2020-11-19'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: food_ingredient_parser
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.6
 platform: ruby
 authors:
 - wvengen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-25 00:00:00.000000000 Z
+date: 2020-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: treetop
@@ -50,6 +50,7 @@ files:
 - lib/food_ingredient_parser/loose/transform/amount.rb
 - lib/food_ingredient_parser/loose/transform/amount_from_name.treetop
 - lib/food_ingredient_parser/loose/transform/handle_missing_name.rb
+- lib/food_ingredient_parser/loose/transform/split_e_numbers.rb
 - lib/food_ingredient_parser/strict/grammar.rb
 - lib/food_ingredient_parser/strict/grammar/amount.treetop
 - lib/food_ingredient_parser/strict/grammar/common.treetop
@@ -86,8 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Parser for ingredient lists found on food products.