textutils 1.3.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8d8339bc5ddee05b87cc13348f12756866bb708f
-  data.tar.gz: d363527756ea6b4e345cb0563b2558aa81717d75
+  metadata.gz: 0db02f95a1da720a3778b0e1dc3636f7318e6fa6
+  data.tar.gz: c3420d50bc0c25e3d7e6ced6b9959c01770d471b
 SHA512:
-  metadata.gz: e47a8c0a60b9d91de8e7d9c8e9b620cd599ee004fa7d1b3b9e597e02c667aa9095a6e4be7595d6a980b66eb512b4804e03ea5ea4f393d7f4294ea6f15b2f76c9
-  data.tar.gz: 0e084073ee1e7cfad863122c82bcb4285c910017f75055dfaa52c93decdad24d5c98d7d3d2639b1450a289362ef1dcb03e9dce3b491c8224ba5ec2f5f077a072
+  metadata.gz: 690e28c6bb15160be57234c3092ef2db54eabfa5d7b66f6ed58dbcd6d5b876451654833c9c0334d55921571fb8498942ae02e8475d437dbd3944ba78afd340ab
+  data.tar.gz: 7348393e33bd3813c55c5b1360ec524836aa1cffb545e53597b599d89f009505c64560df2f705fac44c687644cfcc404aa2a653de8fdc8c6b054f0ae192c144f

data/Manifest.txt

@@ -1,61 +1,63 @@
-HISTORY.md
-Manifest.txt
-README.md
-Rakefile
-lib/textutils.rb
-lib/textutils/classifier.rb
-lib/textutils/core_ext/array.rb
-lib/textutils/core_ext/file.rb
-lib/textutils/core_ext/time.rb
-lib/textutils/filter/code_filter.rb
-lib/textutils/filter/comment_filter.rb
-lib/textutils/filter/erb_django_filter.rb
-lib/textutils/filter/erb_filter.rb
-lib/textutils/filter/string_filter.rb
-lib/textutils/helper/address_helper.rb
-lib/textutils/helper/date_helper.rb
-lib/textutils/helper/hypertext_helper.rb
-lib/textutils/helper/tag_helper.rb
-lib/textutils/helper/title_helper.rb
-lib/textutils/helper/unicode_helper.rb
-lib/textutils/helper/value_helper_i.rb
-lib/textutils/helper/value_helper_ii.rb
-lib/textutils/helper/value_helper_iii_numbers.rb
-lib/textutils/helper/xml_helper.rb
-lib/textutils/page.rb
-lib/textutils/parser/name_parser.rb
-lib/textutils/parser/name_tokenizer.rb
-lib/textutils/patterns.rb
-lib/textutils/reader/block_reader.rb
-lib/textutils/reader/code_reader.rb
-lib/textutils/reader/fixture_reader.rb
-lib/textutils/reader/hash_reader.rb
-lib/textutils/reader/line_reader.rb
-lib/textutils/reader/tree_reader.rb
-lib/textutils/reader/values_reader.rb
-lib/textutils/sanitizier.rb
-lib/textutils/title.rb
-lib/textutils/title_mapper.rb
-lib/textutils/utils.rb
-lib/textutils/version.rb
-test/data/at-austria/1--n-niederoesterreich/orte.txt
-test/data/cl_all.txt
-test/data/de-deutschland/3--by-bayern/4--oberfranken/orte.txt
-test/data/de-deutschland/3--by-bayern/4--oberfranken/orte_ii.txt
-test/data/de-deutschland/orte.txt
-test/data/feedburner.txt
-test/helper.rb
-test/test_address_helper.rb
-test/test_asciify.rb
-test/test_block_reader.rb
-test/test_fixture_reader.rb
-test/test_hypertext_helper.rb
-test/test_slugify.rb
-test/test_taglist.rb
-test/test_title_finder.rb
-test/test_title_helper.rb
-test/test_title_mapper.rb
-test/test_tree_reader.rb
-test/test_tree_reader_ii.rb
-test/test_unicode_helper.rb
-test/test_values_reader.rb
+HISTORY.md
+Manifest.txt
+README.md
+Rakefile
+lib/textutils.rb
+lib/textutils/classifier.rb
+lib/textutils/core_ext/array.rb
+lib/textutils/core_ext/file.rb
+lib/textutils/core_ext/time.rb
+lib/textutils/filter/code_filter.rb
+lib/textutils/filter/comment_filter.rb
+lib/textutils/filter/erb_django_filter.rb
+lib/textutils/filter/erb_filter.rb
+lib/textutils/filter/string_filter.rb
+lib/textutils/helper/address_helper.rb
+lib/textutils/helper/date_helper.rb
+lib/textutils/helper/hypertext_helper.rb
+lib/textutils/helper/tag_helper.rb
+lib/textutils/helper/title_helper.rb
+lib/textutils/helper/unicode_helper.rb
+lib/textutils/helper/value_helper_i.rb
+lib/textutils/helper/value_helper_ii.rb
+lib/textutils/helper/value_helper_iii_numbers.rb
+lib/textutils/helper/xml_helper.rb
+lib/textutils/page.rb
+lib/textutils/parser/name_parser.rb
+lib/textutils/parser/name_tokenizer.rb
+lib/textutils/patterns.rb
+lib/textutils/reader/block_reader.rb
+lib/textutils/reader/code_reader.rb
+lib/textutils/reader/fixture_reader.rb
+lib/textutils/reader/hash_reader.rb
+lib/textutils/reader/line_reader.rb
+lib/textutils/reader/tree_reader.rb
+lib/textutils/reader/values_reader.rb
+lib/textutils/sanitizier.rb
+lib/textutils/title.rb
+lib/textutils/title_mapper.rb
+lib/textutils/title_mapper2.rb
+lib/textutils/utils.rb
+lib/textutils/version.rb
+test/data/at-austria/1--n-niederoesterreich/orte.txt
+test/data/cl_all.txt
+test/data/de-deutschland/3--by-bayern/4--oberfranken/orte.txt
+test/data/de-deutschland/3--by-bayern/4--oberfranken/orte_ii.txt
+test/data/de-deutschland/orte.txt
+test/data/feedburner.txt
+test/helper.rb
+test/test_address_helper.rb
+test/test_asciify.rb
+test/test_block_reader.rb
+test/test_fixture_reader.rb
+test/test_hypertext_helper.rb
+test/test_slugify.rb
+test/test_taglist.rb
+test/test_title_finder.rb
+test/test_title_helper.rb
+test/test_title_mapper.rb
+test/test_title_mapper2.rb
+test/test_tree_reader.rb
+test/test_tree_reader_ii.rb
+test/test_unicode_helper.rb
+test/test_values_reader.rb

data/Rakefile

@@ -21,7 +21,7 @@ Hoe.spec 'textutils' do
     ['props',    '>=1.1.2'],
     ['logutils', '>=0.6.1'],
     ### 3rd party gems
-    ['rubyzip'],   ## todo/check: make optional -why? why not??
+    ['rubyzip', '>=1.0.0'],   ## note: 1.0 changed to require zip (pre 1.0 was zip/zip); todo/check: make optional -why? why not??
     ['activesupport']    ## todo/check:  really needed? document what methods get used
   ]
 

data/lib/textutils.rb

@@ -66,6 +66,7 @@ require 'textutils/reader/tree_reader'
 require 'textutils/classifier'
 require 'textutils/title'    # title table/mapper/finder utils
 require 'textutils/title_mapper'
+require 'textutils/title_mapper2'
 
 require 'textutils/page'   # for book pages and page templates
 

data/lib/textutils/title_mapper2.rb

@@ -0,0 +1,168 @@
+# encoding: utf-8
+
+
+## see textutils/title.rb
+##   for existing code
+##   move over here
+
+
+module TextUtils
+
+class TitleMapper2      ## todo/check: rename to NameMapper ? why? why not??
+
+  include LogUtils::Logging
+
+  attr_reader :known_titles   ## rename to mapping or mappings or just titles - why? why not?
+
+  ##
+  ##  key:      e.g. augsburg 
+  ##  title:    e.g. FC Augsburg
+  ##  length (of title - not pattern):   e.g. 11   -- do not count dots (e.g. U.S.A. => 3 or 6) why? why not? 
+  MappingStruct =  Struct.new( :key, :title, :length, :pattern)   ## todo/check: use (rename to) TitleStruct - why? why not??
+
+
+  def initialize( records, tag )
+    @known_titles = build_title_table_for( records )   ## build mapping lookup table
+
+    ## todo: rename tag to attrib or attrib_name - why ?? why not ???
+    @tag = tag   # e.g. tag name use for @@brewery@@ @@team@@ etc.
+  end
+
+
+  def map_titles!( line )   ## rename to just map! - why?? why not???
+    begin
+      found = map_title_for!( @tag, line, @known_titles )
+    end while found
+  end
+
+  def find_key!( line )
+    find_key_for!( @tag, line )
+  end
+
+  def find_keys!( line )  # NB: keys (plural!) - will return array
+    counter = 1
+    keys = []
+
+    key = find_key_for!( "#{@tag}#{counter}", line )
+    while key.present?
+      keys << key
+      counter += 1
+      key = find_key_for!( "#{@tag}#{counter}", line )
+    end
+    keys
+  end
+
+
+private
+  def build_title_table_for( records )
+
+    ## build known tracks table w/ synonyms e.g.
+    #
+    # [[ 'wolfsbrug', 'VfL Wolfsburg'],
+    #  [ 'augsburg',  'FC Augsburg'],
+    #  [ 'augsburg',  'Augi2'],
+    #  [ 'augsburg',  'Augi3' ],
+    #  [ 'stuttgart', 'VfB Stuttgart']]
+
+    known_titles = []
+
+    records.each_with_index do |rec,index|
+
+      title_candidates = []
+      title_candidates << rec.title
+
+      title_candidates += rec.synonyms.split('|') if rec.synonyms.present?
+
+
+      ## check if title includes subtitle e.g. Grand Prix Japan (Suzuka Circuit)
+      #  make subtitle optional by adding title w/o subtitle e.g. Grand Prix Japan
+
+      titles = []
+      title_candidates.each do |t|
+        titles << t
+        if t =~ /\(.+\)/
+          extra_title = t.gsub( /\(.+\)/, '' ) # remove/delete subtitles
+          # note: strip leading n trailing withspaces too!
+          #  -- todo: add squish or something if () is inline e.g. leaves two spaces?
+          extra_title.strip!
+          titles << extra_title
+        end
+      end
+
+      titles.each do |t|
+        m = MappingStruct.new
+        m.key     = rec.key
+        m.title   = t
+        m.length  = t.length
+        ## note: escape for regex plus allow subs for special chars/accents
+        m.pattern = TextUtils.title_esc_regex( t )
+      
+        known_titles << m
+      end
+
+      logger.debug "  #{rec.class.name}[#{index+1}] #{rec.key} >#{titles.join('|')}<"      
+
+      ## NB: only include code field - if defined
+      if rec.respond_to?(:code) && rec.code.present?
+        m = MappingStruct.new
+        m.key     = rec.key
+        m.title   = rec.code
+        m.length  = rec.code.length
+        m.pattern = rec.code   ## note: use code for now as is (no variants allowed fow now)
+
+        known_titles << m      
+      end
+    end
+
+    ## note: sort here by length (largest goes first - best match)
+      #  exclude code and key (key should always go last)
+    known_titles = known_titles.sort { |left,right| right.length <=> left.length }
+    known_titles
+  end
+
+
+  def map_title_for!( tag, line, mappings )
+
+    downcase_tag = tag.downcase
+
+    mappings.each do |mapping|
+      
+      key   = mapping.key
+      value = mapping.pattern
+      ## nb: \b does NOT include space or newline for word boundry (only alphanums e.g. a-z0-9)
+      ## (thus add it, allows match for Benfica Lis.  for example - note . at the end)
+
+      ## check add $ e.g. (\b| |\t|$) does this work? - check w/ Benfica Lis.$
+      regex = /\b#{value}(\b| |\t|$)/   # wrap with world boundry (e.g. match only whole words e.g. not wac in wacker) 
+      if line =~ regex
+        logger.debug "     match for #{downcase_tag}  >#{key}< >#{value}<"
+        # make sure @@oo{key}oo@@ doesn't match itself with other key e.g. wacker, wac, etc.
+        line.sub!( regex, "@@oo#{key}oo@@ " )    # NB: add one space char at end
+        return true    # break out after first match (do NOT continue)
+      end
+    end
+    return false
+  end
+
+
+  def find_key_for!( tag, line )
+    regex = /@@oo([^@]+?)oo@@/     # e.g. everything in @@ .... @@ (use non-greedy +? plus all chars but not @, that is [^@])
+
+    upcase_tag    = tag.upcase
+    downcase_tag  = tag.downcase
+
+    if line =~ regex
+      value = "#{$1}"
+      logger.debug "   #{downcase_tag}: >#{value}<"
+      
+      line.sub!( regex, "[#{upcase_tag}]" )
+
+      return $1
+    else
+      return nil
+    end
+  end # method find_key_for!
+
+
+end # class TitleMapper2
+end # module TextUtils

data/lib/textutils/version.rb

@@ -3,8 +3,8 @@
 module TextUtils
 
   MAJOR = 1 ## todo: namespace inside version or something - why? why not??
-  MINOR = 3
-  PATCH = 1
+  MINOR = 4
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/test_title_mapper2.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_title_mapper2.rb
+
+
+require 'helper'
+
+
+class TestTitleMapper2 < Minitest::Test
+
+  ClubStruct =  Struct.new(:key, :title, :synonyms)
+
+  def test_title_table
+
+    titles_in = [
+      ClubStruct.new( 'barcelona',  'Barcelona', 'FC Barcelona' ),
+      ClubStruct.new( 'espanyol',   'Espanyol',  'RCD Espanyol|Espanyol Barcelona' ),
+      ClubStruct.new( 'sevilla',    'Sevilla',   'Sevilla FC' )
+    ]
+
+    mapper = TextUtils::TitleMapper2.new( titles_in, 'club' )
+    titles_out = mapper.known_titles
+
+    puts 'titles_out:'
+    pp titles_out
+
+    line = "Espanyol Barcelona  1-0  FC Barcelona"
+    mapper.map_titles!( line )
+    puts "=> #{line}"
+
+    club1 = mapper.find_key!( line )
+    club2 = mapper.find_key!( line )
+    puts "=> #{line}"
+
+    assert_equal 'espanyol',  club1
+    assert_equal 'barcelona', club2
+
+    assert true   ## assume everything ok if we get here
+
+  end # method test_title_table
+
+
+end # class TestTitleMapper2

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: textutils
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-15 00:00:00.000000000 Z
+date: 2015-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: props
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.14'
 description: textutils - Text Filters, Helpers, Readers and More
 email: ruby-talk@ruby-lang.org
 executables: []
@@ -103,7 +103,6 @@ extra_rdoc_files:
 - Manifest.txt
 - README.md
 files:
-- ".gemtest"
 - HISTORY.md
 - Manifest.txt
 - README.md
@@ -142,6 +141,7 @@ files:
 - lib/textutils/sanitizier.rb
 - lib/textutils/title.rb
 - lib/textutils/title_mapper.rb
+- lib/textutils/title_mapper2.rb
 - lib/textutils/utils.rb
 - lib/textutils/version.rb
 - test/data/at-austria/1--n-niederoesterreich/orte.txt
@@ -161,6 +161,7 @@ files:
 - test/test_title_finder.rb
 - test/test_title_helper.rb
 - test/test_title_mapper.rb
+- test/test_title_mapper2.rb
 - test/test_tree_reader.rb
 - test/test_tree_reader_ii.rb
 - test/test_unicode_helper.rb
@@ -187,22 +188,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.2
+rubygems_version: 2.2.3
 signing_key: 
 specification_version: 4
 summary: textutils - Text Filters, Helpers, Readers and More
-test_files:
-- test/test_title_finder.rb
-- test/test_fixture_reader.rb
-- test/test_unicode_helper.rb
-- test/test_asciify.rb
-- test/test_tree_reader.rb
-- test/test_title_mapper.rb
-- test/test_values_reader.rb
-- test/test_taglist.rb
-- test/test_hypertext_helper.rb
-- test/test_title_helper.rb
-- test/test_slugify.rb
-- test/test_address_helper.rb
-- test/test_block_reader.rb
-- test/test_tree_reader_ii.rb
+test_files: []