dwc_agent 1.4.1 → 1.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2819869504dd7262508c63978dae9b03cd6380838bf20f1e5ffecb8464b610a
-  data.tar.gz: 3d1afff77e4bea9995cec5a3f72687646cdb6c9f7637bef8444f7a45df4ab556
+  metadata.gz: 1cf058ee8ca5956a4d05ca0122d5ee9a7497b304576bd4af92fd0f0597846a08
+  data.tar.gz: 3ef7bafb6a3bd9b9af826f161448b259739ec791b4158e5704f6d4da98af48fe
 SHA512:
-  metadata.gz: 223cdf7a80a54e102da1f35d0b98d420d5a3b4ca1a11494ac3feca73a80d86f83a092b3c8b5dac1e5079a56cb5c69c3771876a091770cb4c9e45de0fc905d177
-  data.tar.gz: 042d702364e285c9e629180802ecd2f1898af1c50ec6fbdcacefa8f117d734db7843fd17e0f816771fcef47c7839ccfa8014d963f6798eaec25e7a3854b88289
+  metadata.gz: 0be4b1c78b8fbf61c1a5363ce32efa780372b394bf88e9fd5a5fc42be77eea564d865637feaca08b63712d8780c515db2408e01967519fc014a3b6f5468793c9
+  data.tar.gz: 37095ef2920f06e6152aa58aab26034ce5b00d15778a8ae3270c7672c7e53ba2853223f7f67bfbfc5953d06050b8f2a8c8a2a1f0c3e41ba002aa18411a57092c

data/lib/dwc_agent/cleaner.rb

@@ -22,6 +22,10 @@ module DwcAgent
         return blank_name
       end
 
+      if parsed_namae.given && GIVEN_BLACKLIST.any?{ |s| s.casecmp(parsed_namae.given) == 0 }
+        return blank_name
+      end
+
       if parsed_namae.family && parsed_namae.family.length == 3 && parsed_namae.family.count('.') == 1
         return blank_name
       end
@@ -38,9 +42,9 @@ module DwcAgent
         return blank_name
       end
 
-      if parsed_namae.given && 
-         parsed_namae.family && 
-         parsed_namae.family.count(".") > 0 && 
+      if parsed_namae.given &&
+         parsed_namae.family &&
+         parsed_namae.family.count(".") > 0 &&
          parsed_namae.family.length - parsed_namae.family.count(".") <= 3
           given = parsed_namae.given
           family = parsed_namae.family
@@ -48,9 +52,9 @@ module DwcAgent
           parsed_namae.given = family
       end
 
-      if parsed_namae.given && 
-         parsed_namae.family && 
-         parsed_namae.family.length <=3 && 
+      if parsed_namae.given &&
+         parsed_namae.family &&
+         parsed_namae.family.length <=3 &&
          parsed_namae.family == parsed_namae.family.upcase &&
          parsed_namae.given[-1] != "."
           given = parsed_namae.given
@@ -59,9 +63,9 @@ module DwcAgent
           parsed_namae.given = family
       end
 
-      if parsed_namae.given && 
-        (parsed_namae.given == parsed_namae.given.upcase || 
-        parsed_namae.given == parsed_namae.given.downcase) && 
+      if parsed_namae.given &&
+        (parsed_namae.given == parsed_namae.given.upcase ||
+        parsed_namae.given == parsed_namae.given.downcase) &&
         !parsed_namae.given.include?(".") &&
         parsed_namae.given.tr(".","").length >= 4
           parsed_namae.given = NameCase(parsed_namae.given)
@@ -115,8 +119,12 @@ module DwcAgent
         return blank_name
       end
 
+      if !given.nil? && GIVEN_BLACKLIST.any?{ |s| s.casecmp(given) == 0 }
+        return blank_name
+      end
+
       { given: given, family: family, particle: particle }
     end
 
   end
-end
+end

data/lib/dwc_agent/constants.rb

@@ -1,7 +1,7 @@
 module DwcAgent
   STRIP_OUT = %r{
     ^[\[{(]|
-    [\]})]$|
+    [\]})]\??$|
     \s*?\d+\.\d+|
     \b\d+\(?(?i:[[:alpha:]])\)?\b|
     \b[,;]?\s*(?i:et\.?\s+al)\.?|
@@ -13,9 +13,8 @@ module DwcAgent
     \b[,;]?\s*(?i:unkn?own)\b|
     \b[,;]?\s*(?i:n/a)\b|
     \b[,;]?\s*(?i:ann?onymous)\b|
-    \b[,;]?\s*\(?(?i:undetermined|indeterminable|dummy|interim|accession|illegible|scripsit)\)?\b|
+    \b[,;]?\s*\(?(?i:undetermined|indeterminable|dummy|interim|accession|ill(eg|is)ible|scripsit)\)?\b|
     \b[,;]?\s*(?i:importer|gift)\:?\b|
-    \b[,;]?\s*(?i:frère|frere|père|pere|soeur|sister|bro)\.?(\b|\z)|
     \b[,;]?\s*(?i:string)\b|
     \b[,;]?\s*(?i:person\s*string)\b|
     \b[,;]?\s*(?i:colls)\.(\b|\z)|
@@ -48,10 +47,12 @@ module DwcAgent
     \d+\s+(?i:Oct|Octob(er|re))\.?\b|
     \d+\s+(?i:Nov|Novemb(er|re))\.?\b|
     \d+\s+(?i:Dec|D(e|é)cemb(er|re))\.?\b|
+    \b[.-–,;:/]?\s*(?i:Alabama|Alaska|Arizona|Arkansas|California|Colorado|Connecticut|Delaware|Evergreen|Florida|Georgia|Hawaii|Idaho|Illinois|Indiana|Iowa|Kansas|Kentucky|Louisiana|Maine|Maryland|Massachusetts|Michigan|Minnesota|Mississippi|Missouri|Montana|Nebraska|Nevada|New\s+Hampshire|New\s+Jersey|New\s+Mexico|New\s+York|North\s+Carolina|North\s+Dakota|Ohio|Oklahoma|Oregon|Pennsylvania|Portland|Rhode\s+Island|South\s+Carolina|South\s+Dakota|St\s+Petersburg|Tennessee|Texas|Utah|Vermont|Virginia|Washington|West\s+Virginia|Wisconsin|Wyoming)\s+(?i:State)\s*\b|
     (?i:autres?\s+de|probab|likely|possibl(e|y)|doubtful)|
     \b\s*(?i:maybe)\s*\b|
     \b\s*(?i:prob)\.\s*\b|
     \(?[,]?\s*?(?i:(local)?\s?collector|data\s*recorder|netter|(oper|prepar)ator)\(?s?\)?\.?\:?|
+    \b[.-–,;:]?\s*(?i:department|faculty)\s*?(?i:of)?\s*?(?i:entomology|biology|zoology)|
     (?i:fide)\:?\s*\b|
     (?i:game\s+dept)\.?\s*\b|
     (?i:see\s+notes?\s*(inside)?)|
@@ -66,22 +67,24 @@ module DwcAgent
     ORCID|
     MRI(\s|-)PAS|
     urn\:qm\.qld\.gov\.au\:collector|
-    (?i:University\s+of\s+California)\,?\s+?(?i:Berkeley)?|
+    (?i:University\s+of\s+(Southern\s+)?California(,\s+Berkeley)?)|
     (?i:Field\s+Museum\s+of\s+Natural\s+History)|
     (?i:American\s+Museum\s+of\s+Natural\s+History)|
     (?i:The\s+Paleontological\s+Research\s+Institution)|
     (?i:museum\s+victoria)|
+    \b\s*(?i:United\s+States|Russia)\s*\b|
     (?i:revised|photograph|fruits\s+only)|
     -?\s*(?i:sight\s+(id|identifi?cation))\.?\s*\b|
     -?\s*(?i:synonym(y|ie))|
     \b\s*\(?(?i:(fe)?male)\)?\s*\b|
     \b(?i:to\s+(sub)?spp?)\.?|
     (?i:nom\.?\s+rev\.?)|
-    FNA|DAO|HUH|FDNMB|MNHN|PNI|USNM|
+    FNA|DAO|HUH|FDNMB|MNHN|PNI|USNM|ZMUC|CSIRO|
     AFSC\/POLISH\s+SORTING\s+CTR\.?|
     (?i:university|museum|exhibits?)|
     (?i:uqam)|
     (?i:sem\s+(colec?tor|data))|
+    (?i:no\s+coll\.?(ector)?)|
     \b[,;]\s+\d+\z|
     ["!@?]|
     [,]?\d+|
@@ -133,8 +136,6 @@ module DwcAgent
     '|' => ' | ',
     '(' => ' ',
     ')' => ' ',
-    '[' => ' ',
-    ']' => ' ',
     '?' => '',
     '!' => '',
     '=' => '',
@@ -152,10 +153,12 @@ module DwcAgent
   }
 
   PHRASE_SUBS = {
-    'Dr\.' => 'Dr. ',
-    'Mr\.' => 'Mr. ',
-    'Mrs\.' => 'Mrs. ',
-    'Prof\.' => 'Prof. '
+    'dr\.' => 'Dr. ',
+    'mr\.' => 'Mr. ',
+    'mrs\.' => 'Mrs. ',
+    'prof\.' => 'Prof. ',
+    '\, ph\.d\.' => ' Ph.D.',
+    '\, bro\.' => ' Bro.'
   }
 
   COMPLEX_SEPARATORS = %r{
@@ -193,13 +196,15 @@ module DwcAgent
     (?i:geographic)|
     (?i:mus(eum|ée)|universit(y|é|e|at)|college|institute?|acad(e|é)m|school|écol(e|iers?)|laboratoi?r|projec?t|polytech|dep(t|art?ment)|research|clinic|hospital|cientifica|sanctuary|safari)|
     (?i:univ\.)|
-    (?i:graduate|student|estudi?antes?|labo\.|storekeep|supervisor|superint|rcmp|coordinator|minority|fishermen|police|taxonomist|consultant|participante?s?|team|(é|e)quipe|memb(er|re)|crew|group|staff|personnel|family|captain|friends|assistant|worker)|
+    (?i:graduate|student|élèves?|éleveur|étudiants|estudi?antes?|labo\.|storekeep|supervisor|superint|rcmp|coordinator|minority|fishermen|police|taxonomist|consultant|participante?s?|team|(é|e)quipe|memb(er|re)|crew|group|staff|personnel|family|captain|friends|assistant|worker)|
     (?i:non\s+pr(é|e)cis(é|e))|
+    (?i:no\s+consta)|
     (?i:no\s+(agent)?\s?(data|disponible)(\s+available)?)|
     (?i:not?\s+(entered|stated))|
     (?i:nomenclatur(e|al)\s+adjustment)|
     (?i:not\s+available)|
     (?i:ontario|qu(e|é)bec|saskatchewan|new brunswick|sault|newfoundland|assurance|vancouver|u\.?s\.?s\.?r\.?)|
+    (?i:popa\s+observers?)|
     (?i:recreation|culture)|
     (?i:shaped|dark|pale|areas|phase|spotting|interior|between|closer)|
     (?i:soci(e|é)t(y|é)|cent(er|re)|community|history|conservation|conference|assoc|class|commission|consortium|council|club|exposit|alliance|protective|circle)|
@@ -207,7 +212,7 @@ module DwcAgent
     (?i:sequence\s+data)|
     (?i:size|large|colou?r)\s+|
     (?i:skeleton)|
-    (?i:survey|assessment|station|monitor|stn\.|index|project|bureau|engine|exchange|ex(c|k)ursi(e|o|ó)n?|exped\.?|exp(e|i)di(c|t)i(e|o|ó)n?|experiment|explora(d|t)|festival|generation|inventory|marine|service)|
+    (?i:survey|assessment|station|monitor|stn\.|index|project|bureau|engine|(e|é)x?chang(e|é)s?|ex(c|k)ursi(e|o|ó)n?|exped\.?|exp(e|i)di(c|t)i(e|o|ó)n?|experiment|explora(d|t)|festival|generation|inventory|marine|service)|
     (?i:submersible)|
     (?i:synonymy?)|(topo|syn|holo)type|
     (?i:systematic|perspective)|
@@ -226,21 +231,36 @@ module DwcAgent
   }x
 
   FAMILY_BLACKLIST = [
+    "da",
+    "de'",
+    "del",
     "der",
+    "du",
+    "el",
     "van",
     "von",
     "the",
     "of",
-    "curators",
-    "nomenclatural",
     "adjustment",
     "available",
+    "arachnology",
+    "catalogue",
+    "curators",
     "data",
+    "determination",
+    "dissection",
+    "entered",
+    "nomenclatural",
     "orig",
-    "science",
-    "catalogue"
+    "registration",
+    "science"
+  ]
+
+  GIVEN_BLACKLIST = [
+    "not any",
+    "has not"
   ]
 
-  TITLE = /\s*\b(sir|count(ess)?|(gen|adm|col|maj|capt|cmdr|lt|sgt|cpl|pvt|prof|dr|md|ph\.?d|rev|docteur|mme|abbé|ptre)\.?|frère|frere|père|pere|professor|esq\.?)(\s+|$)/i
+  TITLE = /\s*\b(sir|count(ess)?|(gen|adm|col|maj|capt|cmdr|lt|sgt|cpl|pvt|prof|dr|md|ph\.?d|rev|mme|abbé|ptre|bro|esq)\.?|docteur|father|cantor|vicar|père|pastor|rabbi|reverend|pere|soeur|sister|professor)(\s+|$)/i
 
 end

data/lib/dwc_agent/parser.rb

@@ -8,7 +8,7 @@ module DwcAgent
     end
 
     def initialize
-      options = { 
+      options = {
         prefer_comma_as_separator: true,
         separator: SPLIT_BY,
         title: TITLE
@@ -17,12 +17,12 @@ module DwcAgent
 
       @strip_out_regex = Regexp.new STRIP_OUT.to_s
       @residual_terminators_regex = Regexp.new SPLIT_BY.to_s + %r{\s*\z}.to_s
-      @char_subs_regex = Regexp.new [CHAR_SUBS.keys.join('\\')].to_s
-      @phrase_subs_regex = Regexp.new (PHRASE_SUBS.keys.join('|')).to_s
+      @char_subs_regex = Regexp.new [CHAR_SUBS.keys.join].to_s
+      @phrase_subs_regex = Regexp.new((PHRASE_SUBS.keys.join('|')).to_s, Regexp::IGNORECASE)
       @complex_separators_regex = Regexp.new COMPLEX_SEPARATORS.to_s
       @add_separators_regex = Regexp.new %r{(\S{1}\.)([[:alpha:]]{2,})}.to_s
     end
-      
+
     # Parses the passed-in string and returns a list of names.
     #
     # @param names [String] the name or names to be parsed
@@ -30,6 +30,7 @@ module DwcAgent
     def parse(name)
       return [] if name.nil? || name == ""
       name.gsub!(@strip_out_regex, ' ')
+      name.gsub!(/\[|\]/, '')
       name.gsub!(@char_subs_regex, CHAR_SUBS)
       name.gsub!(@phrase_subs_regex, PHRASE_SUBS)
       name.gsub!(@add_separators_regex, '\1 \2')
@@ -41,4 +42,4 @@ module DwcAgent
     end
 
   end
-end
+end

data/lib/dwc_agent/version.rb

@@ -3,7 +3,7 @@ module DwcAgent
 
     MAJOR = 1
     MINOR = 4
-    PATCH = 1
+    PATCH = 6
     BUILD = nil
 
     def self.version

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dwc_agent
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.4.6
 platform: ruby
 authors:
 - David P. Shorthouse
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-25 00:00:00.000000000 Z
+date: 2020-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: namae
@@ -119,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Parse Darwin Core agent terms such as recordedBy and identifiedBy