food_fish_parser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a92347877837b339f13c2140d2955f41410ea7a6258b34a51f1daf35c4009715
+  data.tar.gz: 9d554028e69f5925e747054cd6c13f3742014ec37ec0621ae1b6008f72a0a8fc
+SHA512:
+  metadata.gz: c93ac59e5393093803ad638ab8992deb0a2af35ed7df7f9e3c0d5666d9477d66660f55bce085cea38b9d3a75c7fd6c17f769af56999b15e745cdb55f4540e6ef
+  data.tar.gz: bf20e42335d25ab91068d2dc8bdd6e8db5a0fbc13595297c97587e0c3a8c4daf3b4668eee32742669cfda32c508e8c2d99bf1fb3dc242c705d1efd2bf63e7550

data/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2018 Questionmark
+Copyright (c) 2018 wvengen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+# Food fish parser
+
+[![Gem Version](https://badge.fury.io/rb/food_fish_parser.svg)](https://rubygems.org/gems/food_fish_parser)
+
+Food products with fish in them often list some details about the particular species,
+fishing method and origin. This [Ruby](https://www.ruby-lang.org/) gem and program parses
+the text found on the product and returns a structured representation.
+
+At this moment, the parser mostly recognises Dutch-language text.
+
+Please note that this code is in an early stage of development.
+
+## Installation
+
+```
+gem install food_fish_parser
+```
+
+## Example
+
+```ruby
+require 'food_fish_parser'
+
+s = <<EOT.gsub(/\n/, '').strip
+  zalm (salmo salar), gekweekt in noorwegen, kweekmethode: kooien.pangasius
+  (pangasius spp), gekweekt in vietnam,  kweekmethode: vijver. coquilles
+  (placopecten magellanicus), vangstgebied noordwest atlantische oceaan fao 21,
+  kabeljauw (gadus macrocephalus), vangstgebied stille oceaan fao 67, garnaal
+  (litopenaeus vannamei), gekweekt in ecuador, kweekmethode: vijver.
+EOT
+parser = FoodFishParser::Parser.new
+puts parser.parse(s).to_a.inspect
+```
+Results in a list of detected fishes
+```ruby
+[
+  {
+    :names =>               [{ :common=>"zalm", :latin=>"salmo salar" }],
+    :catch_areas =>         [],
+    :catch_methods =>       [],
+    :aquaculture_areas =>   [{ :text=>"noorwegen", :fao_codes=>[] }],
+    :aquaculture_methods => [{ :text=>"kooien" }]
+  },
+  {
+    :names =>               [{ :common=>"pangasius", :latin=>"pangasius spp" }],
+    :catch_areas =>         [],
+    :catch_methods =>       [],
+    :aquaculture_areas =>   [{ :text=>"vietnam", :fao_codes=>[] }],
+    :aquaculture_methods => [{ :text=>"vijver" }]
+  },
+  {
+    :names =>               [{ :common=>"coquilles", :latin=>"placopecten magellanicus" }],
+    :catch_areas =>         [{ :text=>"noordwest atlantische oceaan", :fao_codes=>["21"] }],
+    :catch_methods =>       [],
+    :aquaculture_areas =>   [],
+    :aquaculture_methods => []
+  },
+  {
+    :names =>               [{ :common=>"kabeljauw", :latin=>"gadus macrocephalus" }],
+    :catch_areas =>         [{ :text=>"stille oceaan", :fao_codes=>["67"] }],
+    :catch_methods =>       [],
+    :aquaculture_areas =>   [],
+    :aquaculture_methods => []
+  },
+  {
+    :names =>               [{ :common=>"garnaal", :latin=>"litopenaeus vannamei" }],
+    :catch_areas =>         [],
+    :catch_methods =>       [],
+    :aquaculture_areas =>   [{ :text=>"ecuador", :fao_codes=>[] }],
+    :aquaculture_methods => [{ :text=>"vijver" }]
+  }
+]
+```
+
+
+## Test data
+
+[`data/fish-ingredient-samples-qm-nl`](data/fish-ingredient-samples-qm-nl) contains about 2k 
+real-world ingredient lists with fish found on the Dutch market. Each line contains one ingredient
+list (newlines are encoded as `\n`, empty lines and those starting with `#` are ignored).
+
+
+## Species
+
+This gem does very basic named entity recognition of fish names. There are more fish names than the
+parser can handle, so the detected fish names are limited to those actually found in packaged food products.
+At the moment only a very limited number of names is detected. To add more, expand the _species-found_ text
+files in [species/](species/) and run `species/species-treetop-gen.sh`. This updates the fish name grammars.
+
+
+## License
+
+This software is distributed under the [MIT license](LICENSE). Data may have a [different license](data/README.md).

data/bin/food_fish_parser

@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+#
+# Parser for food fish lists.
+#
+require 'optparse'
+
+$:.push(File.expand_path(File.dirname(__FILE__) + "/../lib"))
+require 'food_fish_parser'
+
+begin
+  require 'pry'
+  def pp(o, color: true)
+    if color
+      Pry::ColorPrinter.pp(o)
+    else
+      puts(o.inspect)
+    end
+  end
+rescue LoadError
+  # fallback without color printing
+  def pp(o, color: nil)
+    puts(o.inspect)
+  end
+end
+
+def colorize(color, s)
+  if color
+    "\e[#{color}m#{s}\e[0;22m"
+  else
+    s
+  end
+end
+
+def parse_single(s, parsed=nil, parser:, verbosity: 1, print: nil, escape: false, color: false)
+  parsed ||= parser.parse(s)
+
+  return unless print.nil? || (parsed && print == :parsed) || (!parsed && print == :noresult)
+
+  puts colorize(color && "0;32", escape ? s.gsub("\n", "\\n") : s) if verbosity > 0
+
+  if parsed
+    puts(parsed.inspect) if verbosity > 1
+    pp(parsed.to_a, color: color) if verbosity > 0
+    return true
+  else
+    puts "(no result: #{parser.parser.failure_reason})" if verbosity > 0
+    return false
+  end
+end
+
+def parse_file(path, parser:, verbosity: 1, print: nil, escape: false, color: false)
+  count_parsed = count_noresult = 0
+  File.foreach(path) do |line|
+    next if line =~ /^#/    # comment
+    next if line =~ /^\s*$/ # empty line
+
+    line = line.gsub('\\n', "\n").strip
+    parsed = parser.parse(line)
+    count_parsed += 1 if parsed
+    count_noresult += 1 unless parsed
+
+    parse_single(line, parsed, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color)
+  end
+
+  pct_parsed   = 100.0 * count_parsed / (count_parsed + count_noresult)
+  pct_noresult = 100.0 * count_noresult / (count_parsed + count_noresult)
+  puts "parsed #{colorize(color && "1;32", count_parsed)} (#{pct_parsed.round(1)}%), no result #{colorize(color && "1;31", count_noresult)} (#{pct_noresult.round(1)}%)"
+  return count_noresult
+end
+
+verbosity = 1
+files = []
+strings = []
+print = nil
+escape = false
+color = true
+OptionParser.new do |opts|
+  opts.banner = <<-EOF.gsub(/^    /, '')
+    Usage: #{$0} [options] --file|-f <filename>
+           #{$0} [options] --string|-s <text>
+
+  EOF
+
+  opts.on("-f", "--file FILE", "Parse all lines of the file as fish detail text.") {|f| files << f }
+  opts.on("-s", "--string TEXT", "Parse specified fish detail text.") {|s| strings << s }
+
+  opts.on("-q", "--[no-]quiet", "Only show summary.") {|q| verbosity = q ? 0 : 1 }
+  opts.on("-p", "--parsed", "Only show lines that were successfully parsed.") {|p| print = :parsed }
+  opts.on("-n", "--noresult", "Only show lines that had no result.") {|p| print = :noresult }
+  opts.on("-e", "--[no-]escape", "Escape newlines") {|e| escape = !!e }
+  opts.on("-c", "--[no-]color", "Use color") {|e| color = !!e }
+  opts.on("-v", "--[no-]verbose", "Show more data (parsed tree).") {|v| verbosity = v ? 2 : 1 }
+  opts.on(      "--version", "Show program version.") do
+    puts("food_fish_parser v#{FoodFishParser::VERSION}")
+    exit
+  end
+  opts.on("-h", "--help", "Show this help") do
+    puts(opts)
+    exit
+  end
+end.parse!
+
+if strings.any? || files.any?
+  parser = FoodFishParser::Parser.new
+  success = true
+  strings.each {|s| success &= parse_single(s, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
+  files.each   {|f| success &= parse_file(f, parser: parser, verbosity: verbosity, print: print, escape: escape, color: color) }
+  success or exit(1)
+else
+  STDERR.puts("Please specify one or more --file or --string arguments (see --help).")
+end

data/food_fish_parser.gemspec

@@ -0,0 +1,30 @@
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/lib'))
+require 'food_fish_parser/version'
+
+Gem::Specification.new do |s|
+  s.name             = 'food_fish_parser'
+  s.version          = FoodFishParser::VERSION
+  s.date             = FoodFishParser::VERSION_DATE
+  s.summary          = 'Parser for fish details found on food products.'
+  s.authors          = ['wvengen']
+  s.email            = ['dev-ruby@willem.engen.nl']
+  s.homepage         = 'https://github.com/q-m/food-fish-parser-ruby'
+  s.license          = 'MIT'
+  s.description      = <<-EOD
+    Food products that contain fish sometimes indicate details like fishing
+    area, method or aquaculture country. This parser know about various ways
+    this is found on a product package, and returns a structured representation
+    of the fish ingredient details.
+  EOD
+  s.metadata         = {
+    'bug_tracker_uri' => 'https://github.com/q-m/food-fish-parser-ruby/issues',
+    'source_code_uri' => 'https://github.com/q-m/food-fish-parser-ruby',
+  }
+
+  s.files            = `git ls-files *.gemspec lib`.split("\n")
+  s.executables      = `git ls-files bin`.split("\n").map(&File.method(:basename))
+  s.extra_rdoc_files = ['README.md', 'LICENSE']
+  s.require_paths    = ['lib']
+
+  s.add_runtime_dependency 'treetop', '~> 1.6'
+end

data/lib/food_fish_parser.rb

@@ -0,0 +1,2 @@
+require_relative 'food_fish_parser/version'
+require_relative 'food_fish_parser/parser'

data/lib/food_fish_parser/grammar.rb

@@ -0,0 +1,18 @@
+require 'treetop'
+require_relative 'nodes'
+
+# @todo find a way to auto-generate Ruby from Treetop files when building gem,
+#       see https://stackoverflow.com/q/37794587/2866660
+
+# note that the species name files are autogenerated
+Treetop.load File.dirname(__FILE__) + '/grammar/common'
+Treetop.load File.dirname(__FILE__) + '/grammar/fish_name_latin'
+Treetop.load File.dirname(__FILE__) + '/grammar/fish_name_nl'
+Treetop.load File.dirname(__FILE__) + '/grammar/fish_name'
+Treetop.load File.dirname(__FILE__) + '/grammar/words'
+Treetop.load File.dirname(__FILE__) + '/grammar/fao_area'
+Treetop.load File.dirname(__FILE__) + '/grammar/catch_area'
+Treetop.load File.dirname(__FILE__) + '/grammar/catch_method'
+Treetop.load File.dirname(__FILE__) + '/grammar/aquac_area'
+Treetop.load File.dirname(__FILE__) + '/grammar/aquac_method'
+Treetop.load File.dirname(__FILE__) + '/grammar/root'

data/lib/food_fish_parser/grammar/aquac_area.treetop

@@ -0,0 +1,27 @@
+
+module FoodFishParser::Grammar
+  grammar AquacArea
+    include Common
+    include Words
+    include FaoArea
+
+    rule aquac_area_indicator
+      (
+        'uit'i / 'gekweekt in'i / 'gekweekt op'i /
+        'aquacultuurproduct uit'i / 'aquacultuur product uit'i
+      )
+      !char
+      ( ws* ( ':' / '>' ) )?
+    end
+
+    rule aquac_area_content
+      (
+        ( area:( words ) ( ws* comma? ws* fao_area_list_enclosures )? ) /
+        ( fao_area_list_enclosures ws* comma? ws* area:( words ) ) /
+        fao_area_list_enclosures area:''
+      )
+      <AquacAreaNode>
+    end
+
+  end
+end

data/lib/food_fish_parser/grammar/aquac_method.treetop

@@ -0,0 +1,18 @@
+module FoodFishParser::Grammar
+  grammar AquacMethod
+    include Common
+    include Words
+
+    rule aquac_method_indicator
+      ( 'wijze van vangst en kweekmethode: aquacultuur:'i / 'kweekmethoden'i / 'kweekmethode'i )
+      !char
+      ( ws* ( ':' / '>' ) )?
+    end
+
+    rule aquac_method_content
+      words
+      <AquacMethodNode>
+    end
+
+  end
+end

data/lib/food_fish_parser/grammar/catch_area.treetop

@@ -0,0 +1,32 @@
+module FoodFishParser::Grammar
+  grammar CatchArea
+    include Common
+    include Words
+    include FaoArea
+
+    rule catch_area_indicator
+      ( 'wild'i ws* )?
+      (
+        'gevangen'i ws+ ( 'in'i / 'op'i ) /
+        'visgebied'i / 'vangstgebied'i / 'vangsgebied'i /
+        'betrapt bij'i
+      )
+      !char
+      ( ws* ( ':' / '>' ) )?
+    end
+
+    rule catch_area_indicator_short
+      catch_area_indicator /
+      ( 'in'i / 'op'i ) !char ( ws* ':' )?
+    end
+
+    rule catch_area_content
+      (
+        ( area:( words_no_with ) ( ws* comma? ws* fao_area_list_enclosures )? ) /
+        ( fao_area_list_enclosures ws* comma? ws* area:( words_no_with ) ) /
+        fao_area_list_enclosures area:''
+      )
+      <CatchAreaNode>
+    end
+  end
+end

data/lib/food_fish_parser/grammar/catch_method.treetop

@@ -0,0 +1,24 @@
+module FoodFishParser::Grammar
+  grammar CatchMethod
+    include Common
+    include Words
+
+    rule catch_method_indicator
+      ( 'wild'i ws* )?
+      ( 'gevangen'i ws+ with / 'vangstmethode'i / 'vangsmethode'i )
+      !char
+      ( ws* ( ':' / '>' ) )?
+    end
+
+    rule catch_method_indicator_short
+      catch_method_indicator /
+      with ( ws* ':' )?
+    end
+
+    rule catch_method_content
+      words_no_in_on
+      <CatchMethodNode>
+    end
+
+  end
+end

data/lib/food_fish_parser/grammar/common.treetop

@@ -0,0 +1,38 @@
+module FoodFishParser::Grammar
+  grammar Common
+
+    # whitespace
+    rule ws
+      [ \t]
+    end
+
+    rule char
+      [[:alnum:]] / [-]
+    end
+
+    rule comma
+      ','
+    end
+
+    rule dash
+      [-֊ ‐ ‑ ‒ – — ― ﹘﹣－]
+    end
+
+
+    rule and
+      ( 'and'i / 'en'i / 'und'i ) !char / '&'
+    end
+
+    rule or
+      ( 'or'i / 'of'i / 'oder'i ) !char / '/'
+    end
+
+    rule and_or
+      ( ( 'and/or'i / 'en/of'i ) !char ) / and / or
+    end
+
+    rule with
+      ( 'met'i / 'd.m.v.'i '.'? / 'with'i ) !char
+    end
+  end
+end

data/lib/food_fish_parser/grammar/fao_area.treetop

@@ -0,0 +1,60 @@
+module FoodFishParser::Grammar
+  grammar FaoArea
+    include Common
+
+    rule fao_area_list_enclosures
+      ( '(' ws* fao_area_list ws* ')' ) /
+      ( '|' ws* fao_area_list ) /
+      fao_area_list
+    end
+
+    rule fao_area_list
+      fao_area_indicator ws*
+      ':'? ws*
+      fao_area_code
+      (
+        ( '/' fao_area_code )+ /
+        ( ( ',' ws* fao_area_code )+ ws* comma? ws* and ws+ fao_area_code ) /
+        ( ',' ws* fao_area_code )+
+      )?
+    end
+
+    rule fao_area_indicator
+      ( 'FAO'i / 'FA0'i )
+      ( ( dash / ws+ ) 'gebied'i )? ( ws* 'nr'i '.'? )?
+    end
+
+    rule fao_area_code
+      fao_area_major_code
+      (
+        ( ws* '(' ws* fao_area_sub_range ws* ')' ) /
+        ( fao_area_sub_range )
+      )?
+      <FaoAreaCodeNode>
+    end
+
+    rule fao_area_major_code
+      ( '0' [0-9] [0-9] ) / ( [0-9] [0-9] )
+    end
+
+    rule fao_area_sub_range
+      fao_area_sub_code
+      ( ws* dash ws* fao_area_sub_code )?
+    end
+
+    rule fao_area_sub_code
+      (
+        ( ( dash / '/' / ws* )? [ivxIVX]+ ) /
+        ( dash [0-9] [0-9] )
+      )
+      fao_area_suffix?
+      (
+        ws* '(' ws* [[:digit:]]+ ( ws* dash ws* [[:digit:]]+ )? ws* ')'
+      )?
+    end
+
+    rule fao_area_suffix
+      [abcdABCD]
+    end
+  end
+end

data/lib/food_fish_parser/grammar/fish_name.treetop

@@ -0,0 +1,21 @@
+module FoodFishParser::Grammar
+  grammar FishName
+    include Common
+    include FishNameLatin
+    include FishNameNL
+
+    rule fish_name
+      (
+        fish_name_nl ws* '(' ws* fish_name_latin ws* ')' /
+        fish_name_nl /
+        fish_name_latin
+      )
+      <FishNameNode>
+    end
+
+    rule fish_name_list
+      fish_name ( ws+ and_or ws+ fish_name )*
+    end
+
+  end
+end

data/lib/food_fish_parser/grammar/fish_name_latin.treetop

@@ -0,0 +1,19 @@
+# autogenerated by species-treetop-gen-latin.rb on 2020-03-17
+module FoodFishParser::Grammar
+  grammar FishNameLatin
+    include Common
+
+    rule fish_name_latin
+      fish_name_latin_first ( ws+ fish_name_latin_second )?
+      <FishNameLatinNode>
+    end
+
+    rule fish_name_latin_first
+      'zygochlamys'i / 'zeus'i / 'xiphopenaeus'i / 'xiphias'i / 'undaria'i / 'ulva'i / 'trichiurus'i / 'trachurus'i / 'todarodes'i / 'thunnus'i / 'theragra'i / 'stolephorus'i / 'sprattus'i / 'spirulina'i / 'sparus'i / 'solea'i / 'sepiella'i / 'sepia'i / 'sebastes'i / 'scomber'i / 'sardinella'i / 'sardina'i / 'salmo'i / 'saccharina'i / 'reinhardtius'i / 'psetta'i / 'procambarus'i / 'portunus'i / 'porphyra'i / 'pollachius'i / 'pleuronectes'i / 'pleoticus'i / 'placopecten'i / 'phymatolithon'i / 'perna'i / 'penaeus'i / 'penaeidae'i / 'pelvetia'i / 'pecten'i / 'patinopecten'i / 'parapenaeopsis'i / 'paralomis'i / 'paphia'i / 'pangasius'i / 'pandalus'i / 'palmaria'i / 'pagellus'i / 'pacifische'i / 'ovalipes'i / 'ostrea'i / 'oreochromis'i / 'oncorhynchus'i / 'octopus'i / 'nephrops'i / 'nemipterus'i / 'nelumbo'i / 'mytilus'i / 'mulinia'i / 'micromesistius'i / 'metapenaeus'i / 'merluccius'i / 'merlangius'i / 'melanogrammus'i / 'macruronus'i / 'macrocystis'i / 'lophius'i / 'loligo'i / 'litopenaeus'i / 'lithodes'i / 'limanda'i / 'lethrinus'i / 'lepidotrigla'i / 'lepidopsetta'i / 'lates'i / 'laminaria'i / 'katsuwonus'i / 'illex'i / 'homarus'i / 'himanthalia'i / 'haematococcus'i / 'gracilaria'i / 'gelidium'i / 'gadus'i / 'fucus'i / 'euthynnus'i / 'ensis'i / 'engraulis'i / 'dunaliella'i / 'dosidicus'i / 'dicentrarchus'i / 'crassostrea'i / 'crangon'i / 'clupea'i / 'clarias'i / 'chondrus'i / 'chlorella'i / 'cerastoderma'i / 'caulerpa'i / 'ascophyllum'i / 'anguilla'i / 'anadara'i / 'alle'i / 'alaria'i / 'acipenser'i
+    end
+
+    rule fish_name_latin_second
+      'yessoensis'i / 'vulgaris'i / 'virens'i / 'vesiculosus'i / 'verrucosa'i / 'vannamei'i / 'undulata'i / 'umbilicalis'i / 'tenera'i / 'stylifera'i / 'sprattus'i / 'spp.'i / 'spp'i / 'solea'i / 'scombrus'i / 'santolla'i / 'salina'i / 'salar'i / 'ringens'i / 'pyrifera'i / 'pyrenoidosa'i / 'punctatus'i / 'productus'i / 'pluvialis'i / 'platessa'i / 'platensis'i / 'piscatorius'i / 'pinnatifida'i / 'pilchardus'i / 'pelamis'i / 'pelagicus'i / 'patagonica'i / 'pangasius'i / 'palmata'i / 'pacificus'i / 'officinalis'i / 'ocellatus'i / 'nucifera'i / 'novaezelandiae'i / 'norvegicus'i / 'nodosum'i / 'niloticus'i / 'nerka'i / 'mykiss'i / 'murphyi'i / 'muelleri'i / 'morhua'i / 'monodon'i / 'monoceros'i / 'microptera'i / 'merluccius'i / 'merlangus'i / 'merguiensis'i / 'maximus'i / 'maxima'i / 'marinus'i / 'magellanicus'i / 'macrocephalus'i / 'limanda'i / 'lepturus'i / 'lentillifera'i / 'latissima'i / 'lactuca'i / 'labrax'i / 'kroyeri'i / 'kisutch'i / 'keta'i / 'kabeljauw'i / 'jordani'i / 'japonicus'i / 'japonica'i / 'hippoglossoides'i / 'hexodon'i / 'harengus'i / 'gueldenstaedtii'i / 'granulosa'i / 'gorbuscha'i / 'gladius'i / 'gigas'i / 'gibbosa'i / 'gariepinus'i / 'galloprovincialis'i / 'faber'i / 'esculenta'i / 'encrasicolus'i / 'elongata'i / 'edulis'i / 'edule'i / 'directus'i / 'digitata'i / 'crispus'i / 'crangon'i / 'clarkii'i / 'chilensis'i / 'chalcogramma'i / 'capensis'i / 'canaliculus'i / 'canaliculata'i / 'calcareum'i / 'borealis'i / 'bogaraveo'i / 'bilineata'i / 'australis'i / 'aurata'i / 'argentinus'i / 'antiquata'i / 'anguilla'i / 'anchoita'i / 'americanus'i / 'alle'i / 'albacares'i / 'alalunga'i / 'aeglefinus'i
+    end
+  end
+end

data/lib/food_fish_parser/grammar/fish_name_nl.treetop

@@ -0,0 +1,27 @@
+# autogenerated by species-treetop-gen-nl.rb on 2020-03-17
+module FoodFishParser::Grammar
+  grammar FishNameNL
+    include Common
+
+    rule fish_name_nl
+      ( fish_name_nl_area ws+ )? ( fish_name_nl_attr ws* )? fish_name_nl_name fish_name_nl_suffix?
+      <FishNameCommonNode>
+    end
+
+    rule fish_name_nl_area
+      'pacifische'i / 'indische'i / 'groenlandse'i / 'atlantische'i / 'argentijnse'i / 'alaska'i
+    end
+
+    rule fish_name_nl_attr
+      'zwarte'i / 'zwart'i / 'witte'i / 'witpoot'i / 'wit'i / 'roze'i / 'rood'i / 'rode'i / 'rivier'i / 'pijl'i / 'kleine'i / 'klein'i / 'grote'i / 'groot'i / 'groene'i / 'groen'i / 'doorn'i / 'coho'i / 'chum'i / 'blauwe'i / 'blauw'i
+    end
+
+    rule fish_name_nl_name
+      'zonnevis'i / 'zeewolf'i / 'zeesnoek'i / 'zeekreeft'i / 'zeeforel'i / 'zeebaars'i / 'zalm'i / 'wijting'i / 'weekdieren'i / 'weekdier'i / 'vintonijn'i / 'tonijn'i / 'tong'i / 'tilapia'i / 'tarbot'i / 'tapijtschelp'i / 'sprot'i / 'spie'i / 'snotolf'i / 'snoekbaars'i / 'snoek'i / 'skipjack tonijn'i / 'schol'i / 'schelvis'i / 'schelpen'i / 'schelp'i / 'schar'i / 'sardines'i / 'regenboogforel'i / 'raat'i / 'poon'i / 'pollak'i / 'pangasius'i / 'paling'i / 'oogtonijn'i / 'mul'i / 'mosselen'i / 'mossel'i / 'meerval'i / 'mantelschelp'i / 'makreel'i / 'lom'i / 'leng'i / 'kreeft'i / 'koolvis'i / 'kokkel'i / 'karper'i / 'kabeljauw'i / 'hondstong'i / 'hoki'i / 'heilbot'i / 'heek'i / 'hake'i / 'haai'i / 'ha'i / 'gruis'i / 'griet'i / 'geep'i / 'geelvintonijn'i / 'garnalen'i / 'garnaal'i / 'fint'i / 'coquilles'i / 'cocquilles'i / 'botervis'i / 'bot'i / 'beekridder'i / 'barracuda'i / 'baars'i / 'arkschelp'i / 'ansjovis'i / 'albacore tonijn'i
+    end
+
+    rule fish_name_nl_suffix
+      'vlees'i / 'ringen'i / 'ring'i / 'filets'i / 'filet'i
+    end
+  end
+end

data/lib/food_fish_parser/grammar/root.treetop

@@ -0,0 +1,55 @@
+module FoodFishParser::Grammar
+  grammar Root
+    include Common
+    include FishName
+    include CatchArea
+    include CatchMethod
+    include AquacArea
+    include AquacMethod
+
+    rule root
+      fishes:(
+        ( fish ( ws* and_or ws* fish )+ ) /
+        ( fish ( ws* ( '.' / comma ) ws* fish )+ ) /
+        fish
+      )
+      ( ws* '.' )?
+      <RootNode>
+    end
+
+    rule fish
+      (
+        ( fish_name_list ( ws* ( comma / ':' ) )? ws+ fish_catch_info ) /
+        ( fish_name_list ( ws* ( comma / ':' ) )? ws+ fish_aquac_info ) /
+        fish_name_list /
+        fish_catch_info /
+        fish_aquac_info
+      )
+      <FishNode>
+    end
+
+    rule fish_catch_info
+      (
+        catch_method_indicator ws* catch_method_content
+        ( ( ws* comma )? ws+ catch_area_indicator_short ws* catch_area_content )?
+      ) / (
+        catch_area_indicator ws* catch_area_content
+        ( ( ws* comma )? ws+ catch_method_indicator_short ws* catch_method_content )?
+      )
+    end
+
+    rule fish_aquac_info
+      (
+        aquac_area_indicator ws* aquac_area_content
+          ws* '.' ws* aquac_method_indicator ws* aquac_method_content
+      ) / (
+        aquac_area_indicator ws* aquac_area_content
+          ( ( ws* comma )? ws+ aquac_method_indicator ws* aquac_method_content )?
+      ) / (
+        aquac_method_indicator ws* aquac_method_content
+          ( ( ws* comma )? ws+ aquac_area_indicator ws* aquac_area_content )?
+      )
+    end
+
+  end
+end

data/lib/food_fish_parser/grammar/words.treetop

@@ -0,0 +1,52 @@
+module FoodFishParser::Grammar
+  grammar Words
+    include Common
+    include FishNameLatin
+    include FishNameNL
+
+    rule word
+      word_abbr / '(sub)'i? !words_to_avoid char+
+    end
+
+    rule words
+      word ( word_sep word )*
+    end
+
+    rule words_no_in_on
+      !( 'in'i / 'op'i !char ) word ( word_sep !( 'in'i / 'op'i !char ) word )*
+    end
+
+    rule words_no_with
+      !with word ( word_sep !with word )*
+    end
+
+    rule word_sep
+      ( ws* ( comma / '/' ) ws* ) / ws+
+    end
+
+    rule word_abbr
+      ( [a-zA-Z] '.' )+ [a-zA-Z] / [a-zA-Z] '.' ( [a-zA-Z] '.' )+ ![a-zA-Z]
+    end
+
+    # these words should not be considered, because they indicate a new section
+    rule words_to_avoid
+      (
+        fish_name_latin /
+        fish_name_nl /
+        'gevangen'i /
+        'visgebied'i /
+        'vangstgebied'i /
+        'vangstmethode'i /
+        'vangsmethode'i /
+        'betrapt'i /
+        'gekweekt'i /
+        'kweekmethode'i /
+        'kweekmethoden'i /
+        'd.m.v'i '.'? /
+        'FAO'i /
+        'FA0'i
+      )
+      ![[:alpha:]]
+    end
+  end
+end

data/lib/food_fish_parser/nodes.rb

@@ -0,0 +1,91 @@
+require 'treetop/runtime'
+
+# Needs to be in grammar namespace so Treetop can find the nodes.
+module FoodFishParser
+  module Grammar
+
+    # Additions for Treetop nodes, include this in other nodes where needed.
+    module SyntaxNodeAdditions
+      def to_a_deep(n, cls)
+        if n.is_a?(cls)
+          [n]
+        elsif n.nonterminal?
+          n.elements.map {|m| to_a_deep(m, cls) }.flatten(1).compact
+        end
+      end
+    end
+
+    # Root object, contains everything else.
+    module RootNode
+      include SyntaxNodeAdditions
+      def to_a
+        to_a_deep(fishes, FishNode).map(&:to_h)
+      end
+    end
+
+    module FishNode
+      include SyntaxNodeAdditions
+      def to_h
+        {
+          names: to_a_deep(self, FishNameNode).map(&:to_h),
+          catch_areas: to_a_deep(self, CatchAreaNode).map(&:to_h),
+          catch_methods: to_a_deep(self, CatchMethodNode).map(&:to_h),
+          aquaculture_areas: to_a_deep(self, AquacAreaNode).map(&:to_h),
+          aquaculture_methods: to_a_deep(self, AquacMethodNode).map(&:to_h),
+        }
+      end
+    end
+
+    module FishNameNode
+      include SyntaxNodeAdditions
+      def to_h
+        {
+          common: to_a_deep(self, FishNameCommonNode).first&.text_value,
+          latin: to_a_deep(self, FishNameLatinNode).first&.text_value
+        }
+      end
+    end
+
+    module FishNameCommonNode; end
+    module FishNameLatinNode; end
+
+    module CatchAreaNode
+      include SyntaxNodeAdditions
+      def to_h
+        {
+          text: area.text_value,
+          fao_codes: to_a_deep(self, FaoAreaCodeNode).map(&:text_value)
+        }
+      end
+    end
+
+    module FaoAreaCodeNode; end
+
+    module CatchMethodNode
+      def to_h
+        {
+          text: self.text_value
+        }
+      end
+    end
+
+    module AquacAreaNode
+      include SyntaxNodeAdditions
+      def to_h
+        {
+          text: area.text_value,
+          fao_codes: to_a_deep(self, FaoAreaCodeNode).map(&:text_value)
+        }
+      end
+    end
+
+    module AquacMethodNode
+      def to_h
+        {
+          text: self.text_value
+        }
+      end
+    end
+
+  end
+end

data/lib/food_fish_parser/parser.rb

@@ -0,0 +1,26 @@
+require_relative 'grammar'
+
+module FoodFishParser
+  class Parser
+
+    # @!attribute [r] parser
+    #   @return [Treetop::Runtime::CompiledParser] low-level parser object
+    #   @note This attribute is there for convenience, but may change in the future. Take care.
+    attr_reader :parser
+
+    # Create a new fish detail parser
+    # @return [FoodFishParser::Parser]
+    def initialize
+      @parser = Grammar::RootParser.new
+    end
+
+    # Parse food fish text into a structured representation.
+    #
+    # @return [FoodFishParser::Grammar::RootNode] structured representation of fish details
+    # @note Unrecognized options are passed to Treetop, but this is not guarenteed to remain so forever.
+    def parse(s, **options)
+      @parser.parse(s, **options)
+    end
+
+  end
+end

data/lib/food_fish_parser/version.rb

@@ -0,0 +1,4 @@
+module FoodFishParser
+  VERSION      = '0.1.0'
+  VERSION_DATE = '2020-03-17'
+end

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification
+name: food_fish_parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- wvengen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-03-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: treetop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+description: |2
+      Food products that contain fish sometimes indicate details like fishing
+      area, method or aquaculture country. This parser know about various ways
+      this is found on a product package, and returns a structured representation
+      of the fish ingredient details.
+email:
+- dev-ruby@willem.engen.nl
+executables:
+- food_fish_parser
+extensions: []
+extra_rdoc_files:
+- README.md
+- LICENSE
+files:
+- LICENSE
+- README.md
+- bin/food_fish_parser
+- food_fish_parser.gemspec
+- lib/food_fish_parser.rb
+- lib/food_fish_parser/grammar.rb
+- lib/food_fish_parser/grammar/aquac_area.treetop
+- lib/food_fish_parser/grammar/aquac_method.treetop
+- lib/food_fish_parser/grammar/catch_area.treetop
+- lib/food_fish_parser/grammar/catch_method.treetop
+- lib/food_fish_parser/grammar/common.treetop
+- lib/food_fish_parser/grammar/fao_area.treetop
+- lib/food_fish_parser/grammar/fish_name.treetop
+- lib/food_fish_parser/grammar/fish_name_latin.treetop
+- lib/food_fish_parser/grammar/fish_name_nl.treetop
+- lib/food_fish_parser/grammar/root.treetop
+- lib/food_fish_parser/grammar/words.treetop
+- lib/food_fish_parser/nodes.rb
+- lib/food_fish_parser/parser.rb
+- lib/food_fish_parser/version.rb
+homepage: https://github.com/q-m/food-fish-parser-ruby
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/q-m/food-fish-parser-ruby/issues
+  source_code_uri: https://github.com/q-m/food-fish-parser-ruby
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Parser for fish details found on food products.
+test_files: []