rsssf 0.2.0 → 0.3.0

data/lib/rsssf/convert/html_to_txt/replace_a_href.rb

@@ -0,0 +1,85 @@
+module Rsssf
+class PageConverter
+
+ # <a href="#sa">Série A</a><br>
+  # 
+  #  <A href="http://www.rsssf.org/">Rec.Sport.Soccer 
+  #        Statistics Foundation</A> 
+  #  <A href="http://www.rsssfbrasil.com">RSSSF 
+  #    Brazil</A> 
+  #
+  #  and Daniel Dalence (<A
+  #   href="mailto:danielballack@terra.com.br">danielballack@terra.com.br</A>)
+  ##
+  ##
+  ##  empty
+  ##  <a>Primer Descenso – First Relegation</a>
+
+  A_HREF_RE = %r{<A 
+                    (?: 
+                       \s+ HREF [ ]* = 
+                          (?<href>[^>]+?)
+                    )?
+                  >
+                    (?<title>.+?)
+                  <\/A>
+                }imx              
+
+
+def replace_a_href( html )
+  ## remove anchors (a href)
+  #    note: heading 4 includes anchor (thus, let anchors go first)
+  #  note: <a \newline href is used for authors email - thus incl. support for newline as space
+  html.gsub( A_HREF_RE ) do |match|   ## note: use .+? non-greedy match
+    m = Regexp.last_match
+    captures = m.named_captures
+    href  = if m['href']
+               m['href'].gsub( /["']/, '' ).strip   ## remove ("" or '')
+            else
+               nil
+            end
+    title = m['title'].strip   ## note: "save" caputure first; gets replaced by gsub (next regex call)
+
+
+    if href.nil?
+       ## report error - <a>hello</a> is useless
+       puts " replace anchor w/ missing (!!) href (a) >#{title}<"
+      "‹#{squish(title)}›"
+  
+    ## e.g.
+    ##  ‹Larsen23@gmx.de, see page mailto:Larsen23@gmx.de›
+    ##  ‹danielballack@terra.com.br, see page mailto:danielballack@terra.com.br›
+    ##  ‹zja70@aol.com, see page mailto:zja70@aol.com›)
+  
+    elsif href.start_with?( 'mailto:')
+      puts " blank mailto  -  anchor (a) href >#{href}, >#{title}<"
+      '‹mailto›'   ## delete/remove email
+    else
+      puts " replace anchor (a) href >#{href}, >#{title}<"
+
+      ## convert href to xref
+      xref = if href.start_with?('#')    ## in-page ref
+              ", see §#{href[1..-1]}"
+             elsif href.start_with?( /https?:/ )            ## external page ref
+               ## skip - keep empty - why? why not? (or add url domain?)
+               ''
+             else
+               ## hack - check for some custom excludes  
+               if title.start_with?( 'Rec.Sport.Soccer' )
+                    ## skip - keep empty
+                    '' 
+               else   
+                 ## strip (ending)  .htm|html
+                 ", see page #{href.sub( /\.html?$/,'')}"
+               end
+             end
+
+      "‹#{squish(title)}#{xref}›"
+    end
+  end
+end
+
+
+
+end # module PageConverter
+end # module Rsssf

data/lib/rsssf/convert/html_to_txt/replace_a_name.rb

@@ -0,0 +1,87 @@
+module Rsssf
+class PageConverter
+
+
+  # <a name="sa">Série A</a>
+  # <a name="sd">Série D</a> 
+
+  # <A name=about>
+  #  <H2>About this document</H2></A>
+  #   => change to (possible?)
+  #  <H2><A name=about>About this document</A></H2>
+  #
+  #
+  # <h4><a name="cb">Copa do Brasil</a></h4>
+
+   ## note  - for content use non-greedy to allow 
+   ##              match of tags inside content too
+
+  A_NAME_OLD_RE = %r{<A [ ]+ NAME [ ]* =
+                     (?<name>[^>]+?) 
+                  >
+                     (?<title>.+?) 
+                  </A>
+                }imx
+
+
+  A_NAME_RE = %r{<A [ ]+ NAME [ ]* =
+                     (?<name>[^>]+?) 
+                  >
+                }imx
+
+                
+def replace_a_name_old( html )
+  ##
+  ## remove (named) anchors
+  html.gsub( A_NAME_RE ) do |match|   ## note: use .+? non-greedy match
+    m = Regexp.last_match
+    name = m[:name].gsub( /["']/, '' ).strip   ## remove ("" or '')
+    title = m[:title].strip   ## note: "save" caputure first; gets replaced by gsub (next regex call)
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace anchor (a) name >#{name}<, >#{title}<    -    >#{match}<"
+  
+
+   ##
+   ## todo - report WARN if title incl. tags
+   ##    assumes text only for now - why? why not?
+   ##  add a name inside heading !!!
+   ##  do NOT add heading inside a name !!!
+
+    "#{title}  ‹§#{name}›"   ## note - use two spaces min (between title & name)
+  end
+end
+
+
+
+def replace_a_name( html )
+
+  ## note - allows <a name=""> without closing </a>
+  ##    <a name="semi"><H2>Semifinals</H2>
+  ##   always put anchor on its own line for now
+
+  ##
+  ## remove (named) anchors
+  html.gsub( A_NAME_RE ) do |match|   ## note: use .+? non-greedy match
+    m = Regexp.last_match
+    
+    name = m[:name].gsub( /["']/, '' ).strip   ## remove ("" or '')
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace anchor (a) name >#{name}<    -    >#{match}<"
+  
+   ##
+   ## todo - report WARN if title incl. tags
+   ##    assumes text only for now - why? why not?
+   ##  add a name inside heading !!!
+   ##  do NOT add heading inside a name !!!
+
+    "‹§#{name}›"   ## note - use two spaces min (between title & name)
+  end
+end
+
+
+
+
+end # module PageConverter
+end # module Rsssf
+
+

data/lib/rsssf/convert/html_to_txt/replace_heading.rb

@@ -0,0 +1,76 @@
+module Rsssf
+class PageConverter
+
+
+  ## note - for h1,h2,h3,h4,h5,h6
+  ##     use a backref(erence) e.g. \1
+  ##
+  ## note - include leading and trailing spaces (incl. newlines) !!!
+  ##
+  ## note  - for content inside use non-greedy to allow 
+  ##              match of tags inside content too
+  HEADING_RE = %r{ \s*
+                     <H(?<level>[1-6])>
+                       (?<title> .+?)
+                     </H\k<level>>
+                     \s*
+                   }imx
+
+
+
+  ###
+  ##  note - MUST be a one a single line (see make heading for more)   
+  ##           e.g.    "<h#{tag}>#{text}</h#{tag}>"              
+  BOLD_OR_UNDERLINE_LINE_HEADING_RE = %r{^
+                          [ ]*
+                     <H (?<tag> [BU]) >
+                       (?<title> .+?)
+                     </H \k<tag> >
+                         [ ]*
+                          $
+                   }ix
+  
+                 
+  def replace_heading( html )
+     html = html.gsub( HEADING_RE ) do |_|
+        m = Regexp.last_match
+
+        level = m[:level].to_i(10) 
+        title = m[:title]
+
+        puts " replace heading #{level} (h#{level}) >#{title}<"
+
+        ## note: make sure to always add two newlines before and after
+        "\n\n#{'='*level} #{title}\n\n"    
+        
+     end
+
+     html = html.gsub( BOLD_OR_UNDERLINE_LINE_HEADING_RE ) do |_|
+        m = Regexp.last_match
+
+        tag = m[:tag].downcase 
+        title = m[:title]
+
+        ## use heading 5 for b and heading 6 for underline for now
+        ##   maybe later change to custom  ==_ or ==* or such
+        ##     to mark the heading (sourced via bold/underscore) ???
+        level =  if tag == 'b'
+                      5
+                 elsif tag == 'u'
+                      6 
+                 else
+                     raise ArgumentError, "b(old)|u(underscore) tag expected; got #{tag}"
+                 end
+
+        puts " replace #{tag}-heading #{level} (h#{level}) >#{title}<"
+
+        ## note: do NOT add any newlines before and after
+        "#{'='*level} #{title}"    
+     end
+   
+    html
+  end
+
+
+end # module PageConverter
+end # module Rsssf

data/lib/rsssf/convert/html_to_txt/replace_hr.rb

@@ -0,0 +1,25 @@
+module Rsssf
+class PageConverter
+
+
+
+
+HR_LINE_ASCII = "\n\n=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n\n"
+
+def replace_hr( html )
+
+  html = html.gsub( /\s*<HR>\s*/im ) do |match|
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace horizontal rule (hr) - >#{match}<"
+    HR_LINE_ASCII    ## check what hr to use use  - . - . - or =-=-=-= or somehting distinct?
+  end
+ 
+  html
+end
+
+end # module PageConverter
+end # module Rsssf
+
+
+
+

data/lib/rsssf/convert/html_to_txt.rb

@@ -0,0 +1,247 @@
+module Rsssf
+class PageConverter
+
+
+
+
+
+
+ def squish( str )
+    ## squish more than one white space to one space
+    str.gsub( /[ \r\t\n]+/, ' ' )
+ end
+
+
+
+
+def html_to_txt( html, url: )
+
+###
+#   todo: check if any tags (still) present??
+
+
+  ## cut off everything before body
+  ##   
+  ## note - might incl. attributes e.g.
+  ## <body bgcolor="yellow">
+
+  ## record / track (important) edits - sub(stitutions) etc.
+  edits = []
+
+
+  html = html.sub( /.+?
+                      <BODY [^>]*? >
+                      \s*
+                   /xim, 
+                   '' )
+                  
+  ## special case i)   no <body> - cut-off head if present
+  ## cut off everything before <head/>
+  ##   used in braz93.html, braz98.html
+  html = html.sub( /.+?
+                     <\/HEAD>
+                       \s*
+                   /xim, '' )    
+
+  ## special case ii) no <body>, no </head> 
+  ## cut off everything before <head/>
+  ##   used in braz93.html, braz98.html
+  html = html.sub( /.+?
+                     <HEAD\/>
+                       \s*
+                   /xim, '' )    
+
+
+
+
+  ## cut off everything after body (closing)
+  html = html.sub( /<\/BODY>.*/im, '' )
+  
+  ## special case
+  ## cut off everything after </html> (closing)
+  ##   used in braz93.html, braz98.html
+  html = html.sub( /<\/HTML>.*/im, '' )
+
+
+
+
+  ## quick fix
+  ## <title>World Cup 1950 qualifications</title>
+  ## <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-2">
+  
+  ## remove title and meta
+  html = html.sub( /<TITLE>.*?<\/TITLE>/i, '' )
+  html = html.sub( /<META .*?>/i, '' )
+
+
+  ## (auto-)fix known types / errors
+  ##  todo - pass in/along filename/url too - why? why not?
+  html = errata_html( html )
+
+
+
+  ##
+  ##   change ^<b><a name ...></a></b>$  or    <hb>  - heading "bold" - might be h5
+  ##          ^<u><a name ...></a></u>$    to  <hu>  - heading "underscore" - might be h6
+  html, more_edits = make_heading( html )
+  edits += more_edits 
+
+
+
+
+
+
+  ## remove cite
+  html = html.gsub( /<CITE>([^<]+)<\/CITE>/im ) do |_|
+    puts " remove cite >#{$1}<"
+    "#{$1}"
+  end
+
+
+  html = replace_hr( html )
+
+
+
+
+  ## replace break (br)
+  ## note: do NOT use m/multiline for now - why? why not??
+  html = html.gsub( /<BR>\s*/i ) do |match|    ## note: include (swallow) "extra" newline
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace break (br) - >#{match}<"
+    "\n"
+  end
+
+
+
+  
+  html = replace_a_name( html )
+
+  html = replace_a_href( html )
+
+  ## quickfix  remove trailing </a> left possibly by a_name
+  html = html.gsub( /<\/A>/i, '' )
+
+
+
+  ## replace paragrah (p)
+  html = html.gsub( /\s*<P>\s*/im ) do |match|    ## note: include (swallow) "extra" newline
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace paragraph (p) - >#{match}<"
+    "\n\n"
+  end
+  html = html.gsub( /<\/P>/i, '' )  ## replace paragraph (p) closing w/ nothing for now
+
+
+
+ 
+  html = replace_heading( html )
+
+
+  ## remove i(talics)
+  ##    use non-greedy match as default? e.g. .*? - why? why not?
+  ##    or greedy w/  ([^<]+)
+  html = html.gsub( /<I>(.*?)<\/I>/im ) do |_|
+    puts " remove italic (i) >#{$1}<"
+    "#{$1}"
+  end
+
+  html = html.gsub( /<U>(.*?)<\/U>/im ) do |_|
+    puts " remove underline (u) >#{$1}<"
+    "#{$1}"
+  end
+
+  ## remove b   - note: might include anchors (thus, call after anchors)
+  ###   use non-greedy match as default? e.g. .*? - why? why not?
+  ## was - "**#{$1}**"
+  html = html.gsub( /<B>(.*?)<\/B>/im ) do |_|
+    puts " remove bold (b) >#{$1}<"
+    "#{$1}"  
+  end
+
+  ## <strong></strong>
+  html = html.gsub( /<STRONG>(.*?)<\/STRONG>/im ) do |_|
+    puts " remove strong (strong) >#{$1}<"
+    "#{$1}"  
+  end
+
+
+
+  ## replace preformatted (pre)
+  html = html.gsub( /<PRE>|<\/PRE>/i ) do |match|
+    puts " replace preformatted (pre)"
+
+      ## note - replace preformatted blocks
+      ##           with comments 
+      ##  was:
+      ##  ''  # replace w/ nothing for now (keep surrounding newlines)
+      
+     if match.downcase == '<pre>'
+         '<!-- start pre -->'
+     else
+         '<!-- end pre -->'
+     end
+  end
+
+
+=begin
+  puts
+  puts
+  puts "html:"
+  puts html[0..2000]
+  puts "-- snip --"
+  puts html[-1000..-1]   ## print last hundred chars
+=end
+
+
+  html = remove_emails( html )
+
+
+  html = beautify_anchors( html )
+
+
+  ## check for html tags
+  ##  left
+  ##  use scan instead of 
+  html.gsub( /<
+                \/?
+                [A-Z]+ [^>]*
+              > 
+             /xim ) do |match|
+
+          if ['<menu>', '<ul>', '<li>',
+               '</menu>', '</ul>', '</li>'].include?(match.downcase) 
+               ## do nothing
+          else 
+                  msg = "found unprocessed html tag #{match} in >#{url}<"
+                  puts "*** WARN - #{msg}"
+                  log( msg )  ## log too (see log.txt)
+          end 
+          match
+    end
+
+
+  ##
+  ## todo/fix
+  ##    move up-front - kind of preprocessing (not post) - why? why not?
+
+  ## cleanup whitespaces
+  ##   todo/fix:  convert newline in space first
+  ##                and than collapse spaces etc.!!!
+  txt = String.new
+  html.each_line do |line|
+     line = line.gsub( "\t", '  ' ) # replace all tabs w/ two spaces for nwo
+     line = line.rstrip             # remove trailing whitespace (incl. newline/formfeed)
+
+     txt << line
+     txt << "\n"
+  end
+
+  txt = errata_txt( txt )
+
+  [txt, edits]
+end # method html_to_text
+
+
+
+end # module PageConverter
+end # module Rsssf

data/lib/rsssf/download.rb

@@ -1,151 +1,20 @@
 
 module Rsssf
 
-## end_year to slug_year
-##   check if generic rule/convention in use ???
-## 2007-08: tablesd/duit08.html
-## 2008-09: tablesd/duit09.html
-## 2009-10: tablesd/duit2010.html
-## 2010-11: tablesd/duit2011.html
-## 2011-12: tablesd/duit2012.html
-
-
-    ## map country codes to table pages
-    ##   add options about (char) encoding ??? - why? why not?
-  TABLE = {
-    'eng' => ['tablese/eng{year}',   { encoding: 'Windows-1252' } ],
-    'es'  => ['tabless/span{year}',  { encoding: 'Windows-1252' } ],
-    'de'  => ['tablesd/duit{year}', { encoding: 'Windows-1252' } ],
-    'at'  => ['tableso/oost{year}', { encoding: 'Windows-1252' }  ],
-    'br'  => [
-              ->(season) {
-                  ## note: special slug/case for year/season 2000
-                  ##  see rsssf.org/tablesb/brazchamp.html
-                 if season == Season('2000') 
-                   'tablesb/braz-joao{year}'  ## use braz-joao00 - why? why not?
-                 else
-                   'tablesb/braz{year}'
-                 end
-              },  { encoding: 'Windows-1252' } ],
-  }
-
-
-  BASE_URL = "https://rsssf.org"
-
-
-  def self.table_url( code, season: )
-     url, _ = table_url_and_encoding( code, season: season )
-     url 
-  end
-
-  def self.table_url_and_encoding( code, season: )
-     season = Season( season )
-
-     table = TABLE[ code.downcase ]
-     tmpl     = table[0]
-     tmpl     = tmpl.call( season )  if tmpl.is_a?(Proc)  ## check for proc
-   
-     opts     = table[1] || {}
-     encoding = opts[:encoding]  || 'UTF-8'
-
-
-     slug =  if season.end_year < 2010   ## cut off all digits (only keep last two)s
-                 ##  convert end_year to string with leading zero
-                 '%02d' % (season.end_year % 100)  ## e.g. 00 / 01 / 99 / 98 / 11 / etc.
-              else
-                '%4d' % season.end_year
-              end
-
-     tmpl = tmpl.sub( '{year}', slug )
-     url = "#{BASE_URL}/#{tmpl}.html"
-
-     [url, encoding]
-  end
-
-
-  def self.download_table( code, season: )
-     url, encoding = table_url_and_encoding( code, season: season )
-
-     download_page( url, encoding: encoding )
-  end
-
-
   def self.download_page( url, encoding: )
- 
+
     ## note: assume plain 7-bit ascii for now
-    ##  -- assume rsssf uses ISO_8859_15 (updated version of ISO_8859_1) 
+    ##  -- assume rsssf uses ISO_8859_15 (updated version of ISO_8859_1)
     ###-- does NOT use utf-8 character encoding!!!
     response = Webget.page( url, encoding: encoding )  ## fetch (and cache) html page (via HTTP GET)
 
     ## note: exit on get / fetch error - do NOT continue for now - why? why not?
     exit 1   if response.status.nok?    ## e.g.  HTTP status code != 200
-  
+
 
     puts "html:"
-    html =  response.text( encoding: encoding )    
+    html =  response.text( encoding: encoding )
     pp html[0..400]
     html
   end
 end  # module Rsssf
-
-
-
-__END__
-
-1998-99: tablesd/duit99.html
-1999-00: tablesd/duit00.html      ## use 1999-2000  - why?? why not??
-2000-01: tablesd/duit01.html
-2001-02: tablesd/duit02.html
-2002-03: tablesd/duit03.html
-2003-04: tablesd/duit04.html
-2004-05: tablesd/duit05.html
-2005-06: tablesd/duit06.html
-2006-07: tablesd/duit07.html
-2007-08: tablesd/duit08.html
-2008-09: tablesd/duit09.html
-2009-10: tablesd/duit2010.html
-2010-11: tablesd/duit2011.html
-2011-12: tablesd/duit2012.html
-2012-13: tablesd/duit2013.html
-2013-14: tablesd/duit2014.html
-2014-15: tablesd/duit2015.html
-
-
-2010-11: tableso/oost2011.html
-2011-12: tableso/oost2012.html
-2012-13: tableso/oost2013.html
-2013-14: tableso/oost2014.html
-2014-15: tableso/oost2015.html
-2015-16: tableso/oost2016.html
-
-2011: tablesb/braz2011.html  !! Windows-1252
-2012: tablesb/braz2012.html  !! Windows-1252
-2013: tablesb/braz2013.html  !! Windows-1252
-2014: tablesb/braz2014.html  !! Windows-1252
-2015: tablesb/braz2015.html  !! Windows-1252
-2016: tablesb/braz2016.html  !! Windows-1252
-2017: tablesb/braz2017.html  !! Windows-1252
-2018: tablesb/braz2018.html  !! Windows-1252
-2019: tablesb/braz2019.html  !! Windows-1252
-2020: tablesb/braz2020.html  !! Windows-1252   ## 2020/21  - extended for corona
-2021: tablesb/braz2021.html  !! Windows-1252
-2022: tablesb/braz2022.html  !! Windows-1252
-2023: tablesb/braz2023.html  !! Windows-1252
-2024: tablesb/braz2024.html  !! Windows-1252
-
-2010-11: tablese/eng2011.html  !! Windows-1252
-2011-12: tablese/eng2012.html  !! Windows-1252
-2012-13: tablese/eng2013.html  !! Windows-1252
-2013-14: tablese/eng2014.html  !! Windows-1252
-2014-15: tablese/eng2015.html  !! Windows-1252
-2015-16: tablese/eng2016.html  !! Windows-1252
-2016-17: tablese/eng2017.html  !! Windows-1252
-2017-18: tablese/eng2018.html  !! Windows-1252
-2018-19: tablese/eng2019.html  !! Windows-1252
-2019-20: tablese/eng2020.html  !! Windows-1252
-2020-21: tablese/eng2021.html  !! Windows-1252
-2021-22: tablese/eng2022.html  !! Windows-1252
-2022-23: tablese/eng2023.html  !! Windows-1252
-2023-24: tablese/eng2024.html  !! Windows-1252
-
-