textutils 0.9.8 → 0.9.9

data/Manifest.txt

@@ -32,9 +32,10 @@ lib/textutils/reader/values_reader.rb
 lib/textutils/reader/values_reader_v2.rb
 lib/textutils/sanitizier.rb
 lib/textutils/title.rb
+lib/textutils/title_mapper.rb
 lib/textutils/utils.rb
 lib/textutils/version.rb
-test/cl_all.txt
+test/data/cl_all.txt
 test/helper.rb
 test/test_address_helper.rb
 test/test_asciify.rb

data/lib/textutils.rb

@@ -63,6 +63,7 @@ require 'textutils/reader/fixture_reader'
 
 require 'textutils/classifier'
 require 'textutils/title'    # title table/mapper/finder utils
+require 'textutils/title_mapper'
 
 require 'textutils/page'   # for book pages and page templates
 

data/lib/textutils/reader/code_reader.rb

@@ -6,15 +6,30 @@ class CodeReader
 
   include LogUtils::Logging
 
-  def initialize( path )
-    @path   = path
-
+  def self.from_file( path )
     ## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
     ## - see textutils/utils.rb
-    
-    @code  = File.read_utf8( @path )
+    code = File.read_utf8( path )
+    self.from_string( code )
+  end
+
+  def self.from_string( code )
+    CodeReader.new( code: code )
   end
 
+
+  def initialize( arg )
+    if arg.is_a?( String )  ## old style (deprecated) - pass in filepath as string
+      path = arg
+      logger.info "CodeReader.new - deprecated API - use CodeReader.from_file() instead"
+      @code = File.read_utf8( path )
+    else   ## assume it's a hash
+      opts = arg
+      @code = opts[:code]
+    end
+  end
+
+
   def eval( klass )
     klass.class_eval( @code )
 
@@ -26,4 +41,4 @@ class CodeReader
     # end
   end
 
-end # class CodeReader
+end # class CodeReader

data/lib/textutils/reader/hash_reader.rb

@@ -7,14 +7,28 @@ class HashReader
 
   include LogUtils::Logging
 
-  def initialize( path )
-    @path = path
-
+  def self.from_file( path )
     ## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
     ## - see textutils/utils.rb
-    
-    text = File.read_utf8( @path )
-   
+    text = File.read_utf8( path )
+    self.from_string( text )
+  end
+
+  def self.from_string( text )
+    HashReader.new( text: text )
+  end
+
+  def initialize( arg )
+
+    if arg.is_a?( String )  ## old style (deprecated) - pass in filepath as string
+      path = arg
+      logger.info "HashReader.new - deprecated API - use HashReader.from_file() instead"
+      text = File.read_utf8( path )
+    else   ## assume it's a hash
+      opts = arg
+      text = opts[:text]
+    end
+
     ### hack for syck yaml parser (e.g.ruby 1.9.2) (cannot handle !!null)
     ##   change it to !null to get plain nil
     ##   w/ both syck and psych/libyml

data/lib/textutils/reader/hash_reader_v2.rb

@@ -32,7 +32,7 @@ class HashReaderV2
 
   def each
     path          = "#{include_path}/#{name_real_path}.yml"
-    reader        = HashReader.new( path )
+    reader        = HashReader.from_file( path )
 
     logger.info "parsing data '#{name}' (#{path})..."
 
@@ -46,7 +46,7 @@ class HashReaderV2
 
   def each_typed
     path          = "#{include_path}/#{name_real_path}.yml"
-    reader        = HashReader.new( path )
+    reader        = HashReader.from_file( path )
 
     logger.info "parsing data '#{name}' (#{path})..."
 

data/lib/textutils/reader/line_reader.rb

@@ -1,60 +1,58 @@
 # encoding: utf-8
 
-##
-## fix/todo: move to/merge into LineReader itself
-#   e.g. use  fromString c'tor ??? or similar??
-
 # fix: move into TextUtils namespace/module!!
 
 
 class StringLineReader
+  ## fix/todo:
+  ##   remove - deprecated/obsolete - do NOT use
+  ##   use LineReader.from_string
 
   include LogUtils::Logging
 
-  def initialize( data )
-    @data = data
+  def initialize( text )
+    logger.info "StringLineReader.new - deprecated API - use LineReader.from_string() instead"
+    @reader = LineReader.from_string( text )
   end
 
-
   def each_line
-    @data.each_line do |line|
-  
-      if line =~ /^\s*#/
-        # skip komments and do NOT copy to result (keep comments secret!)
-        logger.debug 'skipping comment line'
-        next
-      end
-        
-      if line =~ /^\s*$/ 
-        # kommentar oder leerzeile überspringen 
-        logger.debug 'skipping blank line'
-        next
-      end
-
-      # remove leading and trailing whitespace
-      line = line.strip
- 
+    @reader.each_line do |line|    
       yield( line )
     end # each lines
   end # method each_line
-
 end
 
 
+
 class LineReader
 
   include LogUtils::Logging
 
-  def initialize( path )
-    @path = path
-
+  def self.from_file( path )
     ## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
     ## - see textutils/utils.rb
-    @data = File.read_utf8( @path )
+    text = File.read_utf8( path )
+    self.from_string( text )
+  end
+
+  def self.from_string( text )
+    LineReader.new( text: text )
+  end
+
+
+  def initialize( arg )
+    if arg.is_a?( String )  ## old style (deprecated) - pass in filepath as string
+      path = arg
+      logger.info "LineReader.new - deprecated API - use LineReader.from_file() instead"
+      @text = File.read_utf8( path )
+    else   ## assume it's a hash
+      opts = arg
+      @text = opts[:text]
+    end
   end
 
   def each_line
-    @data.each_line do |line|
+    @text.each_line do |line|
 
       # comments allow:
       # 1) #####  (shell/ruby style)

data/lib/textutils/reader/line_reader_v2.rb

@@ -32,7 +32,7 @@ class LineReaderV2
 
   def each_line
     path          = "#{include_path}/#{name_real_path}.txt"
-    reader        = LineReader.new( path )
+    reader        = LineReader.from_file( path )
 
     logger.info "parsing data '#{name}' (#{path})..."
 

data/lib/textutils/reader/values_reader.rb

@@ -55,7 +55,7 @@ class ValuesReader
   def initialize( arg, more_attribs={} )
     @more_attribs = more_attribs
     
-    ### todo/fix: rename @data to @text !!!!
+    ### todo/fix: rename @text to @text !!!!
 
     if arg.is_a?( String )  ## old style (deprecated) - pass in filepath as string
       path = arg
@@ -66,14 +66,14 @@ class ValuesReader
       #   check  what is the best convention (follow  ???)
       if path =~ /\n/m
         logger.info "ValuesReader.new - deprecated API - use ValuesReader.from_string() instead"
-        @data = path.dup   # make a duplicate ?? why? why not?
+        @text = path.dup   # make a duplicate ?? why? why not?
       else
         logger.info "ValuesReader.new - deprecated API - use ValuesReader.from_file() instead"
-        @data = File.read_utf8( @path )
+        @text = File.read_utf8( @path )
       end
     else   ## assume it's a hash
       opts = arg
-      @data = opts[:text]
+      @text = opts[:text]
     end
   end
 
@@ -102,7 +102,7 @@ class ValuesReader
     last_header  = nil
 
 
-    @data.each_line do |line|
+    @text.each_line do |line|
 
       ## allow alternative comment lines
       ## e.g. -- comment or

data/lib/textutils/title.rb

@@ -7,19 +7,31 @@
 #   or rename to KeyMapping?, KeyMapper?, KeyTable? etc.
 
 
+######
+## todo/check:
+###   remove - use TitleMapper instead
+##  deprecated/obsolete - do NOT use will get removed
+
+
 module TextUtils
   module TitleTable
 
 ####
 ## fix: turn it into a class w/ methods
 #
-#e.g  t =TitleTable.new( records, name )  # e.g. name='team'
+#e.g  t =TitleMapper.new( records, name )  # e.g. name='team'
 #  t.map!( line )
 #  t.find_key!( line )
 # etc.
+#
+#  see textutils/title_mapper.rb
+#
+#   deprecate code here!!! - move to new TitleMapper class
 
 
 def build_title_table_for( records )
+    LogUtils::Logger.root.info "  build_title_table_for - deprecated API - use TitleMapper.new instead"
+
     ## build known tracks table w/ synonyms e.g.
     #
     # [[ 'wolfsbrug', [ 'VfL Wolfsburg' ]],
@@ -72,6 +84,8 @@ end
 
 
 def find_key_for!( name, line )
+  LogUtils::Logger.root.info "  find_key_for! #{name} - deprecated API - use TitleMapper.find_key! instead"
+
   regex = /@@oo([^@]+?)oo@@/     # e.g. everything in @@ .... @@ (use non-greedy +? plus all chars but not @, that is [^@])
 
   upcase_name   = name.upcase
@@ -92,6 +106,8 @@ end
 
 
 def find_keys_for!( name, line )  # NB: keys (plural!) - will return array
+  LogUtils::Logger.root.info "  find_keys_for! #{name} - deprecated API - use TitleMapper.find_keys! instead"
+
   counter = 1
   keys = []
 
@@ -109,6 +125,8 @@ end
 
 
 def map_titles_for!( name, line, title_table )
+  LogUtils::Logger.root.info "  map_titles_for! #{name} - deprecated API - use TitleMapper.map_titles! instead"
+
   title_table.each do |rec|
     key    = rec[0]
     values = rec[1]

data/lib/textutils/title_mapper.rb

@@ -0,0 +1,159 @@
+# encoding: utf-8
+
+
+## see textutils/title.rb
+##   for existing code
+##   move over here
+
+####
+## fix: turn it into a class w/ methods
+#
+#e.g  t =TitleMapper.new( records, name )  # e.g. name='team'
+#  t.map!( line )
+#  t.find_key!( line )
+# etc.
+
+
+module TextUtils
+
+class TitleMapper      ## todo/check: rename to NameMapper ? why? why not??
+
+  include LogUtils::Logging
+
+  attr_reader :known_titles   ## rename to mapping or mappings or just titles - why? why not?
+
+  def initialize( records, tag )
+    @known_titles = build_title_table_for( records )   ## build mapping lookup table
+
+    ## todo: rename tag to attrib or attrib_name - why ?? why not ???
+    @tag = tag   # e.g. tag name use for @@brewery@@ @@team@@ etc.
+  end
+
+
+  def map_titles!( line )   ## rename to just map! - why?? why not???
+    @known_titles.each do |rec|
+      key    = rec[0]
+      values = rec[1]
+      map_title_for!( @tag, line, key, values )
+    end
+  end
+
+
+  def find_key!( line )
+    find_key_for!( @tag, line )
+  end
+
+  def find_keys!( line )  # NB: keys (plural!) - will return array
+    counter = 1
+    keys = []
+
+    key = find_key_for!( "#{@tag}#{counter}", line )
+    while key.present?
+      keys << key
+      counter += 1
+      key = find_key_for!( "#{@tag}#{counter}", line )
+    end
+    keys
+  end
+
+
+private
+  def build_title_table_for( records )
+
+    #### fix/todo:
+    ###  reorder - sort by largest strings etc.
+    ##   do NOT use lookup w/ array per key; use 1:1 one key per lookup
+    ##     -> lets us sort by find largest first
+
+
+    ## build known tracks table w/ synonyms e.g.
+    #
+    # [[ 'wolfsbrug', [ 'VfL Wolfsburg' ]],
+    #  [ 'augsburg',  [ 'FC Augsburg', 'Augi2', 'Augi3' ]],
+    #  [ 'stuttgart', [ 'VfB Stuttgart' ]] ]
+
+    known_titles = []
+
+    records.each_with_index do |rec,index|
+
+      title_candidates = []
+      title_candidates << rec.title
+
+      title_candidates += rec.synonyms.split('|') if rec.synonyms.present?
+
+
+      ## check if title includes subtitle e.g. Grand Prix Japan (Suzuka Circuit)
+      #  make subtitle optional by adding title w/o subtitle e.g. Grand Prix Japan
+
+      titles = []
+      title_candidates.each do |t|
+        titles << t
+        if t =~ /\(.+\)/
+          extra_title = t.gsub( /\(.+\)/, '' ) # remove/delete subtitles
+          extra_title.strip!   # strip leading n trailing withspaces too!
+          titles << extra_title
+        end
+      end
+
+
+      ## NB: sort here by length (largest goes first - best match)
+      #  exclude code and key (key should always go last)
+      titles = titles.sort { |left,right| right.length <=> left.length }
+      
+      ## escape for regex plus allow subs for special chars/accents
+      titles = titles.map { |title| TextUtils.title_esc_regex( title )  }
+
+      ## NB: only include code field - if defined
+      titles << rec.code          if rec.respond_to?(:code) && rec.code.present?
+
+      known_titles << [ rec.key, titles ]
+
+      logger.debug "  #{rec.class.name}[#{index+1}] #{rec.key} >#{titles.join('|')}<"
+    end
+
+    known_titles
+  end
+
+
+  def map_title_for!( tag, line, key, values )
+
+    downcase_tag = tag.downcase
+
+    values.each do |value|
+      ## nb: \b does NOT include space or newline for word boundry (only alphanums e.g. a-z0-9)
+      ## (thus add it, allows match for Benfica Lis.  for example - note . at the end)
+
+      ## check add $ e.g. (\b| |\t|$) does this work? - check w/ Benfica Lis.$
+      regex = /\b#{value}(\b| |\t|$)/   # wrap with world boundry (e.g. match only whole words e.g. not wac in wacker) 
+      if line =~ regex
+        logger.debug "     match for #{downcase_tag}  >#{key}< >#{value}<"
+        # make sure @@oo{key}oo@@ doesn't match itself with other key e.g. wacker, wac, etc.
+        line.sub!( regex, "@@oo#{key}oo@@ " )    # NB: add one space char at end
+        return true    # break out after first match (do NOT continue)
+      end
+    end
+    return false
+  end
+
+
+  def find_key_for!( tag, line )
+    regex = /@@oo([^@]+?)oo@@/     # e.g. everything in @@ .... @@ (use non-greedy +? plus all chars but not @, that is [^@])
+
+    upcase_tag    = tag.upcase
+    downcase_tag  = tag.downcase
+
+    if line =~ regex
+      value = "#{$1}"
+      logger.debug "   #{downcase_tag}: >#{value}<"
+      
+      line.sub!( regex, "[#{upcase_tag}]" )
+
+      return $1
+    else
+      return nil
+    end
+  end # method find_key_for!
+
+
+end # class TitleMapper
+end # module TextUtils

data/lib/textutils/version.rb

@@ -3,7 +3,7 @@ module TextUtils
 
   MAJOR = 0 ## todo: namespace inside version or something - why? why not??
   MINOR = 9
-  PATCH = 8
+  PATCH = 9
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/helper.rb

@@ -6,7 +6,6 @@
 # require 'minitest/unit'
 require 'minitest/autorun'
 
-# include MiniTest::Unit  # lets us use TestCase instead of MiniTest::Unit::TestCase
 
 ## make sure activesupport gets included/required
 # note: just activesupport or active_support will NOT work

data/test/test_address_helper.rb

@@ -3,7 +3,7 @@
 
 require 'helper'
 
-class TestAddressHelper < MiniTest::Unit::TestCase
+class TestAddressHelper < Minitest::Test
 
   def test_normalize_addr
     

data/test/test_asciify.rb

@@ -4,7 +4,7 @@
 require 'helper'
 
 
-class TestAsciify < MiniTest::Unit::TestCase
+class TestAsciify < Minitest::Test
 
   def test_asciify
 

data/test/test_fixture_reader.rb

@@ -1,14 +1,14 @@
 require 'helper'
 
 
-class TestFixtureReader < MiniTest::Unit::TestCase
+class TestFixtureReader < Minitest::Test
 
   def test_read
-    path = "#{TextUtils.root}/test/cl_all.txt"
+    path = "#{TextUtils.root}/test/data/cl_all.txt"
     puts "[TestFixtureReader.test_read] path: #{path}"
-    
-    reader = FixtureReader.new( path )
-    
+
+    reader = FixtureReader.from_file( path )
+
     ary = [
       'europe-champions-league!/leagues',
       'europe-champions-league!/2011_12/cl',

data/test/test_hypertext_helper.rb

@@ -9,7 +9,7 @@
 require 'helper'
 
 
-class TestHypertextHelper < MiniTest::Unit::TestCase
+class TestHypertextHelper < Minitest::Test
 
   include TextUtils::HypertextHelper   #  lets us use textify, etc.
 

data/test/test_slugify.rb

@@ -2,7 +2,7 @@
 
 require 'helper'
 
-class TestSlugify < MiniTest::Unit::TestCase
+class TestSlugify < Minitest::Test
 
   def test_slugify
 

data/test/test_taglist.rb

@@ -4,7 +4,7 @@
 require 'helper'
 
 
-class TestTaglist < MiniTest::Unit::TestCase
+class TestTaglist < Minitest::Test
 
   include TextUtils::ValueHelper   #  lets us use is_taglist?, etc.
 

data/test/test_title_finder.rb

@@ -4,7 +4,7 @@
 require 'helper'
 
 
-class TestTitleFinder < MiniTest::Unit::TestCase
+class TestTitleFinder < Minitest::Test
 
   include TextUtils::ValueHelper   #  lets us use find_grade, etc.
 

data/test/test_title_helper.rb

@@ -3,7 +3,7 @@
 require 'helper'
 
 
-class TestTitleHelper < MiniTest::Unit::TestCase
+class TestTitleHelper < Minitest::Test
 
   def test_title_to_key
     

data/test/test_title_mapper.rb

@@ -4,7 +4,7 @@
 require 'helper'
 
 
-class TestTitleMapper < MiniTest::Unit::TestCase
+class TestTitleMapper < Minitest::Test
 
   WineryStruct =  Struct.new(:key, :title, :synonyms)
 
@@ -32,7 +32,8 @@ class TestTitleMapper < MiniTest::Unit::TestCase
       ['andreaspolsterer', [ 'Weingut Andreas B\. Polsterer', 'Andreas B\. Polsterer \(1970\)', 'Andreas B\. Polsterer' ]]
     ]
 
-    titles_out = TextUtils.build_title_table_for( titles_in )
+    mapper = TextUtils::TitleMapper.new( titles_in, 'winery' )
+    titles_out = mapper.known_titles
 
     puts 'titles_out:'
     pp titles_out

data/test/test_unicode_helper.rb

@@ -8,7 +8,7 @@
 
 require 'helper'
 
-class TestUnicodeHelper < MiniTest::Unit::TestCase
+class TestUnicodeHelper < Minitest::Test
 
   def test_convert_unicode_dashes
     

data/test/test_values_reader.rb

@@ -8,7 +8,7 @@
 
 require 'helper'
 
-class TestValuesReader < MiniTest::Unit::TestCase
+class TestValuesReader < Minitest::Test
 
   def test_header
     txt =<<EOS
@@ -28,7 +28,7 @@ Schwechater Zwickl,      5.4%, 12.5°, 45.0 kcal/100ml
 
 EOS
 
-    reader = ValuesReader.new( txt )
+    reader = ValuesReader.from_string( txt )
 
     i = 0
     reader.each_line do |attribs, values|
@@ -80,7 +80,7 @@ EOS
 
     pp txt
 
-    reader = ValuesReader.new( txt )
+    reader = ValuesReader.from_string( txt )
 
     i = 0
     reader.each_line do |attribs, values|
@@ -146,7 +146,7 @@ weitra, Weitra Bräu Bierwerkstatt|Brauerei Weitra *, 1321, www.bierwerkstatt.at
 egger, Privatbrauerei Fritz Egger **, 1978, www.egger-bier.at, 3105 Unterradlberg // Tiroler Straße 18
 EOS
 
-    reader = ValuesReader.new( txt )
+    reader = ValuesReader.from_string( txt )
 
     i = 0
     reader.each_line do |attribs, values|
@@ -236,7 +236,7 @@ EOS
   brands: Zwettler
 EOS
 
-    reader = ValuesReader.new( txt )
+    reader = ValuesReader.from_string( txt )
 
     i = 0
     reader.each_line do |attribs, values|
@@ -283,7 +283,7 @@ manunited,   Manchester United|Man Utd|Manchester U.,   MUN, city:manchester
 liverpool,   Liverpool|Liverpool FC|FC Liverpool,       LIV, city:liverpool
 EOS
 
-    reader = ValuesReader.new( txt )
+    reader = ValuesReader.from_string( txt )
 
     i = 0
     reader.each_line do |attribs, values|
@@ -318,7 +318,7 @@ Ottakringer (Gold Fassl) Pils,       4.6 %, 11.2°
 Ottakringer (Gold Fassl) Pur {Bio},  5.2 %, 11.8°, bio
 EOS
 
-    reader = ValuesReader.new( txt )
+    reader = ValuesReader.from_string( txt )
 
     i = 0
     reader.each_line do |attribs, values|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: textutils
 version: !ruby/object:Gem::Version
-  version: 0.9.8
+  version: 0.9.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-06 00:00:00.000000000 Z
+date: 2014-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: props
-  requirement: &72274350 !ruby/object:Gem::Requirement
+  requirement: &79823580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *72274350
+  version_requirements: *79823580
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &72273880 !ruby/object:Gem::Requirement
+  requirement: &79823050 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *72273880
+  version_requirements: *79823050
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &72273580 !ruby/object:Gem::Requirement
+  requirement: &79822760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *72273580
+  version_requirements: *79822760
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &72273200 !ruby/object:Gem::Requirement
+  requirement: &79822540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *72273200
+  version_requirements: *79822540
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &72272730 !ruby/object:Gem::Requirement
+  requirement: &79822210 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,18 +65,18 @@ dependencies:
         version: '4.0'
   type: :development
   prerelease: false
-  version_requirements: *72272730
+  version_requirements: *79822210
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &72272390 !ruby/object:Gem::Requirement
+  requirement: &79821880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: '3.13'
   type: :development
   prerelease: false
-  version_requirements: *72272390
+  version_requirements: *79821880
 description: textutils - Text Filters, Helpers, Readers and More
 email: ruby-talk@ruby-lang.org
 executables: []
@@ -120,9 +120,10 @@ files:
 - lib/textutils/reader/values_reader_v2.rb
 - lib/textutils/sanitizier.rb
 - lib/textutils/title.rb
+- lib/textutils/title_mapper.rb
 - lib/textutils/utils.rb
 - lib/textutils/version.rb
-- test/cl_all.txt
+- test/data/cl_all.txt
 - test/helper.rb
 - test/test_address_helper.rb
 - test/test_asciify.rb