textutils 1.2.4 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 16e24e7bc0a1004bc3fca49b6f3bfcaa6ce2e5ce
-  data.tar.gz: 6aa074466c9c89b089ecf0ac6d75164050abd0cb
+  metadata.gz: cb4855c8da2e08a23a8c84f436058695ca79b5d2
+  data.tar.gz: 7373f05346e5939481ab569bc411d37b8d155a09
 SHA512:
-  metadata.gz: f386324301ffc37deba32eb32202edf9b8706ff1b14971ba5a41db22a2d9a314be64914b4296e359cea2620a6b54ad89ae1cfa6e7c9fa1322ec4ef29020fb688
-  data.tar.gz: d56300976f712b8bb9de8f1e2e571948f35783090d1d7f1a238736f8efd6a60aba915d5ff807cb942a38f47f80b8d6917e6e1e846371b72b62022c8b3ad029f0
+  metadata.gz: 547e53097dcc8ca0bade46448b7fd26998cc4ff333b33142bddb0db6b8654e004d78498c01bdf27fb9c2b10578e8e5808f89e7aa632ec73ffa1f114dbf693979
+  data.tar.gz: d9d26f569c2ebd76766c6ae8d592e08310f5632c40a78dc80f04d2085b031551a7f61f21cca046bce8fcbd922584a273540346f9d26863b41d6093f956bf828e

data/Manifest.txt

@@ -24,12 +24,14 @@ lib/textutils/helper/value_helper_iii_numbers.rb
 lib/textutils/helper/xml_helper.rb
 lib/textutils/page.rb
 lib/textutils/parser/name_parser.rb
+lib/textutils/parser/name_tokenizer.rb
 lib/textutils/patterns.rb
 lib/textutils/reader/block_reader.rb
 lib/textutils/reader/code_reader.rb
 lib/textutils/reader/fixture_reader.rb
 lib/textutils/reader/hash_reader.rb
 lib/textutils/reader/line_reader.rb
+lib/textutils/reader/tree_reader.rb
 lib/textutils/reader/values_reader.rb
 lib/textutils/sanitizier.rb
 lib/textutils/title.rb
@@ -37,6 +39,9 @@ lib/textutils/title_mapper.rb
 lib/textutils/utils.rb
 lib/textutils/version.rb
 test/data/cl_all.txt
+test/data/de-deutschland/3--by-bayern/4--oberfranken/orte.txt
+test/data/de-deutschland/3--by-bayern/4--oberfranken/orte_ii.txt
+test/data/de-deutschland/orte.txt
 test/data/feedburner.txt
 test/helper.rb
 test/test_address_helper.rb
@@ -49,5 +54,6 @@ test/test_taglist.rb
 test/test_title_finder.rb
 test/test_title_helper.rb
 test/test_title_mapper.rb
+test/test_tree_reader.rb
 test/test_unicode_helper.rb
 test/test_values_reader.rb

data/lib/textutils.rb

@@ -52,8 +52,8 @@ require 'textutils/core_ext/file'
 require 'textutils/core_ext/time'
 require 'textutils/core_ext/array'
 
-
 require 'textutils/parser/name_parser'
+require 'textutils/parser/name_tokenizer'
 
 require 'textutils/reader/code_reader'
 require 'textutils/reader/hash_reader'
@@ -61,6 +61,7 @@ require 'textutils/reader/line_reader'
 require 'textutils/reader/values_reader'
 require 'textutils/reader/fixture_reader'
 require 'textutils/reader/block_reader'
+require 'textutils/reader/tree_reader'
 
 require 'textutils/classifier'
 require 'textutils/title'    # title table/mapper/finder utils
@@ -71,4 +72,4 @@ require 'textutils/page'   # for book pages and page templates
 
 
 # say hello
-puts TextUtils.banner   if $DEBUG || (defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG)
+puts TextUtils.banner   if defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG

data/lib/textutils/parser/name_parser.rb

@@ -2,60 +2,65 @@
 
 # fix: move into TextUtils namespace/module!! ??
 
+class NameParser
 
-class NameTokenizer   ## - rename to NameScanner, NameSplitter, NameSeparator, etc.
-
-  ## split (single) string value into array of names
-  ##   e.g.
-  ##   'München [Munich]'             => ['München', '[Munich]']
-  ##   'Wr. Neustadt | Wiener Neustadt' => ['Wr. Neustadt', 'Wiener Neustadt']
   include LogUtils::Logging
 
-  def tokenize( value )   ## rename to/use split - why? why not??
-    names = []
+  def parse( chunks )
+    ## todo/fix: (re)use nameparser - for now "simple" inline version
+    ##  fix!!! - note: for now lang gets ignored
+    ##  fix: add hanlde
+    ##  Leuven[nl]|Louvain[fr] Löwen[de]
+    ##  Antwerpen[nl]|Anvers[fr] [Antwerp]
+    ##  Brussel[nl]•Bruxelles[fr]   -> official bi-lingual name
+    ##  etc.
 
-    # 1)  split by | (pipe) -- remove leading n trailing whitespaces
-    parts = value.split( /[ \t]*\|[ \t]*/ )
+    ## values - split into names (name n lang pairs)
+    ## note: assumes (default) lang from more_attribs unless otherwise marked e.g. [] assume en etc.
 
-    # 2)  split "inline" translations e.g. München [Munich]
+    ## split chunks into values
+    values = []
+    chunks.each do |chunk|
+      next if chunk.nil? || chunk.blank?  ## skip nil or empty/blank chunks
 
-    ## todo: add support for  Munich [en]  e.g. trailing lang tag
-    ## todo: add support for bullet (official bi-lingual names w/ tags ??) - see brussels - why, why not??
+      parts = chunk.split( '|' )   # 1)  split |
 
-    parts.each do |part|
+      parts.each do |part|
         s = StringScanner.new( part )
         s.skip( /[ \t]+/)   # skip whitespaces
 
         while s.eos? == false
           if s.check( /\[/ )
             ## scan everything until the end of bracket (e.g.])
-            name = s.scan( /\[[^\]]+\]/)
-            ## todo/fix: if name nil - issue warning??
-            #  starting w/ [  but no closing ] found !!!! - possible? fix!!
+            ##  fix!!! - note: for now lang gets ignored
+            value = s.scan( /\[[^\]]+\]/)
+            value = value[1...-1]   # strip enclosing [] e.g. [Bavaria] => Bavaria
           else
             ## scan everything until the begin of bracket (e.g.[)
-            name = s.scan( /[^\[]+/)
-            name = name.rstrip   ## remove trailing spaces (if present)
+            value = s.scan( /[^\[]+/)
+            value = value.strip
           end
-          names << name
+          values << value
 
           s.skip( /[ \t]+/)  # skip whitespaces
-          logger.debug( "[NameTokenizer] eos?: #{s.eos?}, rest: >#{s.rest}<" )
+          logger.debug( "[NameParser] eos?: #{s.eos?}, rest: >#{s.rest}<" )
         end
-    end # each part
-
-    logger.debug( "[NameTokenizer] names=#{names.inspect}")
-    names
-  end # method split
-end # class NameTokenizer
+      end
+    end
 
+    logger.debug( "[NameParser] values=#{values.inspect}")
 
-=begin
-class NameParser
+    names = []
+    values.each do |value|
+      name = value
+      ## todo: split by bullet ? (official multilang name) e.g. Brussel • Bruxelles
+      ## todo: process variants w/ () e.g. Krems (a. d. Donau) etc. ??
+      names << name
+    end
 
-  include LogUtils::Logging
+    logger.debug( "[NameParser] names=#{names.inspect}")
 
-  ## to be done
-  
+    names
+  end # method parse
 end # class NameParser
-=end
+

data/lib/textutils/parser/name_tokenizer.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+# fix: move into TextUtils namespace/module!! ??
+
+
+class NameTokenizer   ## - rename to NameScanner, NameSplitter, NameSeparator, etc.
+
+  ## split (single) string value into array of names
+  ##   e.g.
+  ##   'München [Munich]'             => ['München', '[Munich]']
+  ##   'Wr. Neustadt | Wiener Neustadt' => ['Wr. Neustadt', 'Wiener Neustadt']
+  include LogUtils::Logging
+
+  def tokenize( value )   ## rename to/use split - why? why not??
+    names = []
+
+    # 1)  split by | (pipe) -- remove leading n trailing whitespaces
+    parts = value.split( /[ \t]*\|[ \t]*/ )
+
+    # 2)  split "inline" translations e.g. München [Munich]
+
+    ## todo: add support for  Munich [en]  e.g. trailing lang tag
+    ## todo: add support for bullet (official bi-lingual names w/ tags ??) - see brussels - why, why not??
+
+    parts.each do |part|
+        s = StringScanner.new( part )
+        s.skip( /[ \t]+/)   # skip whitespaces
+
+        while s.eos? == false
+          if s.check( /\[/ )
+            ## scan everything until the end of bracket (e.g.])
+            name = s.scan( /\[[^\]]+\]/)
+            ## todo/fix: if name nil - issue warning??
+            #  starting w/ [  but no closing ] found !!!! - possible? fix!!
+          else
+            ## scan everything until the begin of bracket (e.g.[)
+            name = s.scan( /[^\[]+/)
+            name = name.rstrip   ## remove trailing spaces (if present)
+          end
+          names << name
+
+          s.skip( /[ \t]+/)  # skip whitespaces
+          logger.debug( "[NameTokenizer] eos?: #{s.eos?}, rest: >#{s.rest}<" )
+        end
+    end # each part
+
+    logger.debug( "[NameTokenizer] names=#{names.inspect}")
+    names
+  end # method split
+end # class NameTokenizer
+

data/lib/textutils/reader/tree_reader.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+
+# fix: move into TextUtils namespace/module!!
+
+class TreeReader
+
+  include LogUtils::Logging
+
+  def self.from_file( path )
+    ## nb: assume/enfore utf-8 encoding (with or without BOM - byte order mark)
+    ## - see textutils/utils.rb
+   text = File.read_utf8( path )
+   self.from_string( text )
+  end
+
+  def self.from_string( text )
+    self.new( text )
+  end
+
+  def initialize( text )
+    @text = text
+  end
+
+  TreeItem = Struct.new( :level, :key, :value )
+
+  KEY_REGEX     = /
+                      ([0-9][0-9A-Za-z]*)   ## key starting with a nummer
+                        |
+                      ([a-z]+)   ## key all lowercase e.g. bt,n,etc.
+                        |
+                      ([A-Z]+)   ## key all uppercase e.g. BT,N,etc
+                  /x
+
+  LEVEL_REGEX   = /\.+/     ## e.g. .. or .... etc.
+
+
+  def each_line  
+    stack    = []     # note: last_level  => stack.size; starts w/ 0
+    times    = 2      # assume two indents factor (e.g. .. =2, ....=3 etc. ) for now
+
+    reader = LineReader.from_string( @text )
+    reader.each_line do |line|
+
+      logger.debug "[TreeReader]  line (before) => >#{line}<"
+
+      s = StringScanner.new( line )
+      s.skip( /[ \t]+/ )   # remove whitespace
+
+      key = s.scan( KEY_REGEX )
+      if key
+        s.skip( /[ \t]+/ )   # remove whitespace
+      end
+
+      level_str = s.scan( LEVEL_REGEX )
+      if level_str
+        ## FIX!! todo/check: make sure level_str.size is a multiple of two !! (e.g. 2,4,6,etc.)
+        level = (level_str.size/times)+1
+        s.skip( /[ \t]+/ )   # remove whitespace
+      else
+        level = 1   ## no level found; assume top level (start w/ 1)
+      end
+
+      ## assume rest is record
+      rest = s.rest  ## was: s.scan( /.+/ )
+
+      level_diff = level - stack.size
+
+      if level_diff > 0
+        logger.debug "[TreeReader]    up  +#{level_diff}"
+        ## FIX!!! todo/check/verify/assert: always must be +1
+      elsif level_diff < 0
+        logger.debug "[TreeReader]    down #{level_diff}"
+        level_diff.abs.times { stack.pop }
+        stack.pop
+      else
+        ## same level
+        stack.pop
+      end
+
+      item = TreeItem.new
+      item.level = level
+      item.key   = key
+      item.value = rest
+
+      stack.push( item )
+
+      ## for debugging - show tree item (note) hierarchy 
+      names = stack.map { |it| "(#{it.level}) #{it.value}" }
+      logger.debug "[TreeReader]    #{names.join( ' › ' )}  -- key: >#{key}<, level: >#{level}<, rest: >#{rest}<"
+
+      yield( stack )
+    end
+  end # method each_line
+
+end # class TreeReader
+

data/lib/textutils/version.rb

@@ -3,8 +3,8 @@
 module TextUtils
 
   MAJOR = 1 ## todo: namespace inside version or something - why? why not??
-  MINOR = 2
-  PATCH = 4
+  MINOR = 3
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/data/de-deutschland/3--by-bayern/4--oberfranken/orte.txt

@@ -0,0 +1,103 @@
+2     Bayern
+24    .. Oberfranken
+241   .... Bamberg (Stadt)     ## Kreisfreie Stadt
+      ...... Bamberg
+      ........ Bamberg
+242   .... Bayreuth (Stadt)    ## Kreisfreie Stadt
+      ...... Bayreuth
+      ........ Bayreuth
+243   .... Coburg (Stadt)      ## Kreisfreie Stadt
+      ...... Coburg
+      ........ Coburg 
+244   .... Hof (Stadt)         ## Kreisfreie Stadt
+      ...... Hof
+      ........ Hof
+
+245   .... Bamberg (Land)      ## Landkreis   -- 36 Gemeinden; see de.wikipedia.org/wiki/Landkreis_Bamberg
+             ## 4 Städte
+      ...... Baunach        ## (4013, 30,9 km²)
+      ........ Baunach
+      ...... Hallstadt      ## (8364, 14,5 km²)
+      ........ Hallstadt    ## (7588)
+      ........ Dörfleins    ## (1380)
+      ...... Scheßlitz      ## (7184, 94,9 km²)
+      ........ Scheßlitz
+      ........ Köttensdorf
+      ........ Würgau
+      ...... Schlüsselfeld  ## (5712, 70,2 km²)
+
+             ## 8 Märkte
+      ...... Burgebrach             ## (6553, 87,9 km²)
+      ...... Burgwindheim           ## (1311, 37,4 km²)
+      ...... Buttenheim             ## (3472, 30 km²)
+      ...... Ebrach                 ## (1830, 29,6 km²)
+      ...... Heiligenstadt i. OFr.  ## (3525, 76,7 km²)
+      ........ Heiligenstadt i. OFr.
+      ........ Oberleinleiter
+      ...... Hirschaid              ## (11.919, 41 km²)
+      ...... Rattelsdorf            ## (4568, 39,6 km²)
+      ........ Rattelsdorf
+      ........ Mürsbach
+      ........ Freudeneck
+      ........ Höfen
+      ........ Ebing
+      ...... Zapfendorf             ## (4954, 30,6 km²)
+
+             ## 24 Gemeinden
+      ...... Altendorf              ## (2012, 8,6 km²)
+      ...... Bischberg              ## (6012, 17,5 km²)
+      ...... Breitengüßbach         ## (4586, 16,9 km²)
+      ........ Breitengüßbach
+      ...... Frensdorf              ## (4865, 44 km²)
+      ...... Gerach                 ## (946, 7,8 km²)
+      ...... Gundelsheim            ## (3378, 3,8 km²)
+      ...... Kemmern                ## (2544, 8,3 km²)
+      ........ Kemmern
+      ...... Königsfeld             ## (1335, 42,7 km²)
+      ........ Königsfeld
+      ........ Huppendorf
+      ...... Lauter                 ## (1139, 12,8 km²)
+      ........ Lauter               ## (601)
+      ........ Appendorf            ## (213)
+      ...... Lisberg                ## (1813, 8,4 km²)
+      ...... Litzendorf             ## (6057, 25,9 km²)
+      ........ Litzendorf
+      ........ Schammelsdorf
+      ........ Melkendorf
+      ........ Lohndorf
+      ........ Tiefenellern
+      ...... Memmelsdorf            ## (8854, 26,2 km²)
+      ........ Memmelsdorf
+      ........ Merkendorf
+      ........ Drosendorf      
+      ...... Oberhaid               ## (4590, 27,2 km²)
+      ........ Oberhaid
+      ........ Staffelbach
+      ...... Pettstadt                  ## (1940, 9,9 km²)
+      ...... Pommersfelden              ## (2851, 35,7 km²)
+      ...... Priesendorf                ## (1470, 8,4 km²)
+      ...... Reckendorf                 ## (2033, 13,1 km²)
+      ........ Reckendorf
+      ...... Schönbrunn im Steigerwald  ## (1880, 24,7 km²)
+      ...... Stadelhofen                ## (1250, 41 km²)
+      ........ Stadelhofen
+      ........ Steinfeld
+      ........ Schederndorf
+      ...... Stegaurach                 ## (6842, 23,9 km²)
+      ...... Strullendorf               ## (7807, 31,7 km²)
+      ........ Strullendorf
+      ........ Geisfeld
+      ........ Roßdorf am Forst
+      ...... Viereth-Trunstadt          ## (3562, 15,8 km²)
+      ...... Walsdorf                   ## (2575, 16,2 km²)
+      ...... Wattendorf                 ## (679, 22,2 km²)
+      ........ Wattendorf
+
+246   .... Bayreuth (Land)     ## Landkreis
+247   .... Coburg (Land)       ## Landkreis
+248   .... Forchheim
+249   .... Hof (Land)          ## Landkreis
+24A   .... Kronach
+24B   .... Kulmbach
+24C   .... Lichtenfels
+24D   .... Wunsiedel i. Fichtelgebirge

data/test/data/de-deutschland/3--by-bayern/4--oberfranken/orte_ii.txt

@@ -0,0 +1,17 @@
+2     Bayern
+24    .. Oberfranken
+241   .... Bamberg (Stadt)     ## Kreisfreie Stadt
+      ...... Bamberg
+      ........ Bamberg
+242   .... Bayreuth (Stadt)    ## Kreisfreie Stadt
+      ...... Bayreuth
+      ........ Bayreuth
+
+245   .... Bamberg (Land)      ## Landkreis   -- 36 Gemeinden; see de.wikipedia.org/wiki/Landkreis_Bamberg
+             ## 4 Städte
+      ...... Baunach        ## (4013, 30,9 km²)
+      ........ Baunach
+      ...... Hallstadt      ## (8364, 14,5 km²)
+      ........ Hallstadt    ## (7588)
+      ........ Dörfleins    ## (1380)
+

data/test/data/de-deutschland/orte.txt

@@ -0,0 +1,12 @@
+2     Bayern
+24    .. Oberfranken
+241   .... Bamberg (Stadt)     ## Kreisfreie Stadt
+      ...... Bamberg
+      ........ Bamberg
+
+#####
+# todo: for testing add berlin and some more
+
+9     Berlin
+91    .. Berlin
+

data/test/test_tree_reader.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_tree_reader.rb
+
+
+require 'helper'
+
+class TestTreeReader < MiniTest::Test
+
+  def test_oberfranken
+    reader = TreeReader.from_file( "#{TextUtils.root}/test/data/de-deutschland/3--by-bayern/4--oberfranken/orte.txt" )
+ 
+    reader.each_line do |_|
+      ## do nothing for now
+    end
+    
+    assert true ## assume everything ok if we get here
+  end
+
+  def test_de
+    reader = TreeReader.from_file( "#{TextUtils.root}/test/data/de-deutschland/orte.txt" )
+ 
+    reader.each_line do |_|
+      ## do nothing for now
+    end
+
+    assert true ## assume everything ok if we get here
+  end
+
+end # class TestTreeReader
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: textutils
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.3.0
 platform: ruby
 authors:
 - Gerald Bauer
@@ -130,12 +130,14 @@ files:
 - lib/textutils/helper/xml_helper.rb
 - lib/textutils/page.rb
 - lib/textutils/parser/name_parser.rb
+- lib/textutils/parser/name_tokenizer.rb
 - lib/textutils/patterns.rb
 - lib/textutils/reader/block_reader.rb
 - lib/textutils/reader/code_reader.rb
 - lib/textutils/reader/fixture_reader.rb
 - lib/textutils/reader/hash_reader.rb
 - lib/textutils/reader/line_reader.rb
+- lib/textutils/reader/tree_reader.rb
 - lib/textutils/reader/values_reader.rb
 - lib/textutils/sanitizier.rb
 - lib/textutils/title.rb
@@ -143,6 +145,9 @@ files:
 - lib/textutils/utils.rb
 - lib/textutils/version.rb
 - test/data/cl_all.txt
+- test/data/de-deutschland/3--by-bayern/4--oberfranken/orte.txt
+- test/data/de-deutschland/3--by-bayern/4--oberfranken/orte_ii.txt
+- test/data/de-deutschland/orte.txt
 - test/data/feedburner.txt
 - test/helper.rb
 - test/test_address_helper.rb
@@ -155,6 +160,7 @@ files:
 - test/test_title_finder.rb
 - test/test_title_helper.rb
 - test/test_title_mapper.rb
+- test/test_tree_reader.rb
 - test/test_unicode_helper.rb
 - test/test_values_reader.rb
 homepage: https://github.com/textkit/textutils
@@ -188,6 +194,7 @@ test_files:
 - test/test_fixture_reader.rb
 - test/test_unicode_helper.rb
 - test/test_asciify.rb
+- test/test_tree_reader.rb
 - test/test_title_mapper.rb
 - test/test_values_reader.rb
 - test/test_taglist.rb