food_fish_parser 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a92347877837b339f13c2140d2955f41410ea7a6258b34a51f1daf35c4009715
-  data.tar.gz: 9d554028e69f5925e747054cd6c13f3742014ec37ec0621ae1b6008f72a0a8fc
+  metadata.gz: 83e6a16aa71bcfb8c10737828059764fb46bb855bedef1a682695e3aed9a4b31
+  data.tar.gz: 866d70e749ca95dcd449ac64b2617e6076fac00e2361c5f252272d27a677fa14
 SHA512:
-  metadata.gz: c93ac59e5393093803ad638ab8992deb0a2af35ed7df7f9e3c0d5666d9477d66660f55bce085cea38b9d3a75c7fd6c17f769af56999b15e745cdb55f4540e6ef
-  data.tar.gz: bf20e42335d25ab91068d2dc8bdd6e8db5a0fbc13595297c97587e0c3a8c4daf3b4668eee32742669cfda32c508e8c2d99bf1fb3dc242c705d1efd2bf63e7550
+  metadata.gz: c4dfdf7a85fadc219b423e7d8a95b7d3d0fd4f699657dbcbd4bf43e48b1f89cd4e5975fc48f25c145d56182483a5914fb148475dc50c9d2359f4e3540f7b2ec7
+  data.tar.gz: 65f0b596fa65870a130bbfdd4663086fdd6c46c1d9349a2cb3b3561c43b2d0bd541df924af7df976cb946f98c79e716129c005f671a191766a80075d8aae07f1

data/README.md

@@ -31,7 +31,9 @@ EOT
 parser = FoodFishParser::Parser.new
 puts parser.parse(s).to_a.inspect
 ```
+
 Results in a list of detected fishes
+
 ```ruby
 [
   {
@@ -72,12 +74,98 @@ Results in a list of detected fishes
 ]
 ```
 
+When you have a piece of text and don't know where (or if) any fish details are
+present, you can use the `anywhere` option.
+
+```ruby
+require 'food_fish_parser'
+
+parser = FoodFishParser::Parser.new
+s = "tomaat, vis (zalm (salmo salar) gevangen in Noorwegen), zout"
+puts parser.parse(s, anywhere: true).to_a.inspect
+```
+
+This will find as many occurences as possible. It is assumed that all fish details
+in the text have the same amount of information (so fish name plus catch or aquaculture
+information, or only fish names, or only catch or aquaculture information).
+While the parser would normally return nothing, with `anywhere` it returns:
+
+```ruby
+[
+  {
+    :names               => [{ :common=>"zalm", :latin=>"salmo salar" }],
+    :catch_areas         => [{ :text=>"Noorwegen", :fao_codes=>[] }],
+    :catch_methods       => [],
+    :aquaculture_areas   => [],
+    :aquaculture_methods => []
+  }
+]
+```
+
+Please note that the `anywhere` option can make the parser much slower.
+
+
+## Test tool
+
+The executable `food_fish_parser` is available after installing the gem. If you're
+running from the source tree, use `bin/food_fish_parser` instead.
+
+```
+$ food_fish_parser -h
+Usage: bin/food_fish_parser [options] --file|-f <filename>
+       bin/food_fish_parser [options] --string|-s <text>
+
+    -f, --file FILE                  Parse all lines of the file as fish detail text.
+    -s, --string TEXT                Parse specified fish detail text.
+    -q, --[no-]quiet                 Only show summary.
+    -p, --parsed                     Only show lines that were successfully parsed.
+    -n, --noresult                   Only show lines that had no result.
+    -a, --[no-]anywhere              Search for fish details anywhere in the text.
+    -e, --[no-]escape                Escape newlines
+    -c, --[no-]color                 Use color
+    -v, --[no-]verbose               Show more data (parsed tree).
+        --version                    Show program version.
+    -h, --help                       Show this help
+
+$ food_fish_parser -v -s "salmo salar"
+"salmo salar"
+SyntaxNode+Root6+RootNode+SyntaxNodeAdditions offset=0, "salmo salar" (to_a,to_a_deep):
+  SyntaxNode+Root3 offset=0, "salmo salar" (fish_only_names):
+    SyntaxNode+FishNode+SyntaxNodeAdditions+FishNameList1 offset=0, "salmo salar" (to_h,to_a_deep,fish_name):
+      SyntaxNode+FishNameNode+SyntaxNodeAdditions+FishNameLatin1+FishNameLatinNode offset=0, "salmo salar" (to_h,to_a_deep,fish_name_latin_first):
+        SyntaxNode offset=0, "salmo"
+        SyntaxNode+FishNameLatin0 offset=5, " salar" (fish_name_latin_second):
+          SyntaxNode offset=5, " ":
+            SyntaxNode offset=5, " "
+          SyntaxNode offset=6, "salar"
+      SyntaxNode offset=11, ""
+    SyntaxNode offset=11, ""
+  SyntaxNode offset=11, ""
+  SyntaxNode offset=11, ""
+  SyntaxNode offset=11, ""
+[
+  {
+    :names=>[{:common=>nil, :latin=>"salmo salar"}],
+    :catch_areas=>[],
+    :catch_methods=>[],
+    :aquaculture_areas=>[],
+    :aquaculture_methods=>[]
+  }
+]
+
+$ food_fish_parser -q -f data/test-cases
+parsed 51 (100.0%), no result 0 (0.0%)
+```
+
+If you want to use the output in (shell)scripts, the options `-e -c` may be quite useful.
+
 
 ## Test data
 
-[`data/fish-ingredient-samples-qm-nl`](data/fish-ingredient-samples-qm-nl) contains about 2k 
+[`data/fish-ingredient-samples-qm-nl`](data/fish-ingredient-samples-qm-nl) contains about 2k
 real-world ingredient lists with fish found on the Dutch market. Each line contains one ingredient
-list (newlines are encoded as `\n`, empty lines and those starting with `#` are ignored).
+list (newlines are encoded as `\n`, empty lines and those starting with `#` are ignored). Of those,
+something is returned for 99.8% of them (with the `anywhere` option), but quality varies greatly.
 
 
 ## Species

data/bin/food_fish_parser

@@ -31,8 +31,8 @@ def colorize(color, s)
   end
 end
 
-def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false, color: false)
-  parsed ||= parser.parse(s)
+def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false, color: false, anywhere: false)
+  parsed ||= parser.parse(s, anywhere: anywhere)
 
   return unless print.nil? || (parsed && print == :parsed) || (!parsed && print == :noresult)
 
@@ -48,14 +48,14 @@ def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false
   end
 end
 
-def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: false)
+def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: false, anywhere: false)
   count_parsed = count_noresult = 0
   File.foreach(path) do |line|
     next if line =~ /^#/    # comment
     next if line =~ /^\s*$/ # empty line
 
     line = line.gsub('\\n', "\n").strip
-    parsed = parser.parse(line)
+    parsed = parser.parse(line, anywhere: anywhere)
     count_parsed += 1 if parsed
     count_noresult += 1 unless parsed
 
@@ -72,6 +72,7 @@ verbosity = 1
 files = []
 strings = []
 print = nil
+anywhere = false
 escape = false
 color = true
 OptionParser.new do |opts|
@@ -87,6 +88,7 @@ OptionParser.new do |opts|
   opts.on("-q", "--[no-]quiet", "Only show summary.") {|q| verbosity = q ? 0 : 1 }
   opts.on("-p", "--parsed", "Only show lines that were successfully parsed.") {|p| print = :parsed }
   opts.on("-n", "--noresult", "Only show lines that had no result.") {|p| print = :noresult }
+  opts.on("-a", "--[no-]anywhere", "Search for fish details anywhere in the text.") {|a| anywhere = !!a }
   opts.on("-e", "--[no-]escape", "Escape newlines") {|e| escape = !!e }
   opts.on("-c", "--[no-]color", "Use color") {|e| color = !!e }
   opts.on("-v", "--[no-]verbose", "Show more data (parsed tree).") {|v| verbosity = v ? 2 : 1 }
@@ -103,8 +105,8 @@ end.parse!
 if strings.any? || files.any?
   parser = FoodFishParser::Parser.new
   success = true
-  strings.each {|s| success &= parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
-  files.each   {|f| success &= parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
+  strings.each {|s| success &= parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color, anywhere: anywhere) }
+  files.each   {|f| success &= parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color, anywhere: anywhere) }
   success or exit(1)
 else
   STDERR.puts("Please specify one or more --file or --string arguments (see --help).")

data/food_fish_parser.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.license          = 'MIT'
   s.description      = <<-EOD
     Food products that contain fish sometimes indicate details like fishing
-    area, method or aquaculture country. This parser know about various ways
+    area, method or aquaculture country. This parser knows about various ways
     this is found on a product package, and returns a structured representation
     of the fish ingredient details.
   EOD

data/lib/food_fish_parser/grammar/aquac_area.treetop

@@ -7,13 +7,18 @@ module FoodFishParser::Grammar
 
     rule aquac_area_indicator
       (
-        'uit'i / 'gekweekt in'i / 'gekweekt op'i /
+        'gekweekt in'i / 'gekweekt op'i /
         'aquacultuurproduct uit'i / 'aquacultuur product uit'i
       )
       !char
       ( ws* ( ':' / '>' ) )?
     end
 
+    rule aquac_area_indicator_short
+      aquac_area_indicator /
+      ( 'uit'i ) !char ( ws* ( ':' / '>' ) )?
+    end
+
     rule aquac_area_content
       (
         ( area:( words ) ( ws* comma? ws* fao_area_list_enclosures )? ) /

data/lib/food_fish_parser/grammar/common.treetop

@@ -3,7 +3,7 @@ module FoodFishParser::Grammar
 
     # whitespace
     rule ws
-      [ \t]
+      [ \t\r\n]
     end
 
     rule char

data/lib/food_fish_parser/grammar/fish_name_nl.treetop

@@ -4,12 +4,13 @@ module FoodFishParser::Grammar
     include Common
 
     rule fish_name_nl
+      ( 'verse'i ws+ )?
       ( fish_name_nl_area ws+ )? ( fish_name_nl_attr ws* )? fish_name_nl_name fish_name_nl_suffix?
       <FishNameCommonNode>
     end
 
     rule fish_name_nl_area
-      'pacifische'i / 'indische'i / 'groenlandse'i / 'atlantische'i / 'argentijnse'i / 'alaska'i
+      'pacifische'i / 'noorse'i / 'indische'i / 'groenlandse'i / 'atlantische'i / 'argentijnse'i / 'alaska'i
     end
 
     rule fish_name_nl_attr

data/lib/food_fish_parser/grammar/root.treetop

@@ -7,27 +7,61 @@ module FoodFishParser::Grammar
     include AquacArea
     include AquacMethod
 
+    # Regular root node that requires all text to match fish details.
+    #
+    # Note that here we prefer fish with catch or aquaculture info,
+    # then try fish names, and finally only catch or aquaculture info.
+    #
+    # The assumption is that all declared fish would have the same amount
+    # of information. Hence we first try for all info, then resort to partial
+    # info.
     rule root
-      fishes:(
-        ( fish ( ws* and_or ws* fish )+ ) /
-        ( fish ( ws* ( '.' / comma ) ws* fish )+ ) /
-        fish
+      (
+        ( fish_with_info ( ws* fish_sep ws* fish_with_info )* ) /
+        ( fish_only_names ( ws* fish_sep ws* fish_only_names )* ) /
+        ( fish_only_info ( ws* fish_sep ws* fish_only_info )* )
       )
-      ( ws* '.' )?
+      ws* '.'? ws*
       <RootNode>
     end
 
-    rule fish
+    # Alternate root node that allows fish details to be interspaced with other text.
+    #
+    # Note that this can be a much more expensive operation.
+    rule root_anywhere
+      (
+        ( !fish_with_info . )* ( fish_with_info ( ( !fish_with_info . )* fish_with_info )* ) ( !fish_with_info . )* /
+        ( !fish_only_names . )* ( fish_only_names ( ( !fish_only_names . )+ fish_only_names )* ) ( !fish_only_names . )* /
+        ( !fish_only_info . )* ( fish_only_info ( ( !fish_only_info . )+ fish_only_info )* ) ( !fish_only_info . )*
+      )
+      <RootNode>
+    end
+
+    # separator between fish declarations
+    rule fish_sep
+      and_or / '.' / comma
+    end
+
+    # fish with catch or aquaculture info
+    rule fish_with_info
       (
         ( fish_name_list ( ws* ( comma / ':' ) )? ws+ fish_catch_info ) /
-        ( fish_name_list ( ws* ( comma / ':' ) )? ws+ fish_aquac_info ) /
-        fish_name_list /
-        fish_catch_info /
-        fish_aquac_info
+        ( fish_name_list ( ws* ( comma / ':' ) )? ws+ fish_aquac_info )
       )
       <FishNode>
     end
 
+    # fish names only
+    rule fish_only_names
+      fish_name_list <FishNode>
+    end
+
+    # catch or aquaculture info only (no names)
+    rule fish_only_info
+      ( fish_catch_info / fish_aquac_info ) <FishNode>
+    end
+
+
     rule fish_catch_info
       (
         catch_method_indicator ws* catch_method_content
@@ -47,9 +81,8 @@ module FoodFishParser::Grammar
           ( ( ws* comma )? ws+ aquac_method_indicator ws* aquac_method_content )?
       ) / (
         aquac_method_indicator ws* aquac_method_content
-          ( ( ws* comma )? ws+ aquac_area_indicator ws* aquac_area_content )?
+          ( ( ws* comma )? ws+ aquac_area_indicator_short ws* aquac_area_content )?
       )
     end
-
   end
 end

data/lib/food_fish_parser/grammar/words.treetop

@@ -39,6 +39,7 @@ module FoodFishParser::Grammar
         'vangstmethode'i /
         'vangsmethode'i /
         'betrapt'i /
+        'aquacultuurproduct'i /
         'gekweekt'i /
         'kweekmethode'i /
         'kweekmethoden'i /

data/lib/food_fish_parser/nodes.rb

@@ -19,7 +19,7 @@ module FoodFishParser
     module RootNode
       include SyntaxNodeAdditions
       def to_a
-        to_a_deep(fishes, FishNode).map(&:to_h)
+        to_a_deep(self, FishNode).map(&:to_h)
       end
     end
 

data/lib/food_fish_parser/parser.rb

@@ -16,9 +16,15 @@ module FoodFishParser
 
     # Parse food fish text into a structured representation.
     #
+    # @param s [String] text to parse
+    # @param anywhere [Bool] +false+ assume the text is only fish details, +true+ to search for fish details in the text
     # @return [FoodFishParser::Grammar::RootNode] structured representation of fish details
     # @note Unrecognized options are passed to Treetop, but this is not guarenteed to remain so forever.
-    def parse(s, **options)
+    def parse(s, anywhere: false, **options)
+      if anywhere
+        options = options.merge(root: :root_anywhere, consume_all_input: false)
+      end
+
       @parser.parse(s, **options)
     end
 

data/lib/food_fish_parser/version.rb

@@ -1,4 +1,4 @@
 module FoodFishParser
-  VERSION      = '0.1.0'
-  VERSION_DATE = '2020-03-17'
+  VERSION      = '0.2.0'
+  VERSION_DATE = '2020-03-18'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: food_fish_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - wvengen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-17 00:00:00.000000000 Z
+date: 2020-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: treetop
@@ -26,7 +26,7 @@ dependencies:
         version: '1.6'
 description: |2
       Food products that contain fish sometimes indicate details like fishing
-      area, method or aquaculture country. This parser know about various ways
+      area, method or aquaculture country. This parser knows about various ways
       this is found on a product package, and returns a structured representation
       of the fish ingredient details.
 email: