rsssf 0.0.1 → 0.2.0

data/lib/rsssf/convert.rb

@@ -0,0 +1,495 @@
+
+module Rsssf
+class PageConverter
+ 
+    ## convenience helper
+     def self.convert( html, url: )
+           @@converter ||= new   ## use a "shared" built-in converter
+           @@converter.convert( html, url: url )
+     end
+    
+  ##
+  ##  add anchor: options or such 
+  ##    lets you toggle adding anchors (§premier etc.) - why? why not?  
+  
+  def convert( html, url: )
+    ### todo/fix: first check if html is all ascii-7bit e.g.
+    ## includes only chars from 64 to 127!!!
+  
+    ## normalize newlines
+    ##   remove \r (form feed) used by Windows; just use \n (new line)
+    html = html.gsub( "\r", '' )
+  
+    ## check for html entities
+    html = html.gsub( "ä", 'ä' )
+    html = html.gsub( "ö", 'ö' )
+    html = html.gsub( "ü", 'ü' )
+    html = html.gsub( "Ä", 'Ä' )
+    html = html.gsub( "Ö", 'Ö' )
+    html = html.gsub( "Ü", 'Ü' )
+    html = html.gsub( "ß", 'ß' )
+  
+    ## typos / autofix - keep - why? why not?
+    html = html.gsub( "&oulm;", 'ö' )    ## support typo in entity (ö)
+    html = html.gsub( "¨",  'ü' )    ## support typo in entity (ü) - why? why not?
+    html = html.gsub( "&slig;", "ß" )    ## support typo in entity (ß)
+    html = html.gsub( "&aaacute;", "á" )  ## typo for á 
+   
+
+    html = html.gsub( "É", 'É' )
+    html = html.gsub( "ø", 'ø' )
+    html = html.gsub( "ã", 'ã' )
+    html = html.gsub( "õ", 'õ' )
+    html = html.gsub( "ô", 'ô' )
+
+    entities = %w[
+À   À
+Á   Á
+Â   Â
+Ã   Ã
+Ä   Ä
+Å   Å
+à   à
+á   á
+â   â
+ã   ã
+ä   ä
+å   å
+Æ   Æ
+æ   æ
+ß   ß
+Ç   Ç
+ç   ç
+È   È
+É   É
+Ê   Ê
+Ë   Ë
+è   è
+é   é
+ê   ê
+ë   ë
+Ì   Ì
+Í   Í
+Î   Î
+Ï   Ï
+ì   ì
+í   í
+î   î
+ï   ï
+Ñ   Ñ
+ñ   ñ
+Ò   Ò
+Ó   Ó
+Ô   Ô
+Õ   Õ
+Ö   Ö
+ò   ò
+ó   ó
+ô   ô
+õ   õ
+ö   ö
+Ø   Ø
+ø   ø
+Ù   Ù
+Ú   Ú
+Û   Û
+Ü   Ü
+ù   ù
+ú   ú
+û   û
+ü   ü
+Ý   Ý
+ý   ý
+ÿ   ÿ
+
+<    &lt;
+>    &gt;
+&    &amp;
+©    &copy;
+®    &reg;
+
+Š    &#352; 
+š    &#353; 
+č    &#269; 
+ć    &#263; 
+Ž    &#381;
+’    &#8217;
+  ]
+
+
+
+    entities.each_slice(2) do |str, entity|
+       html = html.gsub( entity, str )
+    end
+
+
+
+    ##############
+    ## check for more entities
+    ##   limit &---; to length 10 - why? why not?
+    html = html.gsub( /&[^; ]{1,10};/) do |match|
+  
+      match =   if match == '&#307;'   ## use like Van D&#307;k  -> Van Dijk
+                  'ij'
+                else
+                  msg = "found unencoded html entity #{match}"
+                  puts "*** WARN - #{msg}"
+                  log( msg )  ## log too (see log.txt)
+
+                  match   ## pass through as is (1:1)
+                end
+  
+      match   
+    end
+    ## todo/fix: add more entities
+  
+###################################
+ ### smart quotes quick fixes   
+### convert all "smart" quote to (standard) single quotes
+##  D´Alessandro   =>  D'Alessandro 
+  
+    html = html.gsub( '´', "'" )
+    
+    html = html.gsub( '’', "'" )
+    html = html.gsub( '‘', "'" )
+    html = html.gsub( '“', '"' )
+    html = html.gsub( '”', '"' )
+
+### convert fancy dashes/hyphens to plain dash/hyphen  
+     html = html.gsub( '–', '-' )
+ 
+
+     
+    txt   = html_to_txt( html )
+  
+    header = <<EOS
+  <!--
+     source: #{url}
+    -->
+  
+EOS
+  
+    header+txt  ## return txt w/ header
+  end  ## method convert
+  
+
+  ## todo/fix - use generic heading regex for all h2/h3/h4 etc.
+  ##  exclude h1 - why? why not?
+  ## note - include leading and trailing spaces !!!
+  ##
+  ## note  - for content use non-greedy to allow 
+  ##              match of tags inside content too
+  HEADING2_RE = %r{ \s*
+                     <H2>
+                       (?<title>.+?)
+                     </H2>
+                     \s*
+                   }imx
+                   
+  HEADING4_RE = %r{ \s*
+                   <H4>
+                     (?<title>.+?)
+                   </H4>
+                   \s*
+                 }imx
+
+  def replace_h2( html )
+     html.gsub( HEADING2_RE ) do |_|
+        m = Regexp.last_match
+        puts " replace heading 2 (h2) >#{m[:title]}<"
+        "\n\n## #{m[:title]}\n\n"    ## note: make sure to always add two newlines
+     end
+  end
+
+  def replace_h4( html )
+    html.gsub( HEADING4_RE ) do |_|
+       m = Regexp.last_match
+       puts " replace heading 4 (h4) >#{m[:title]}<"
+       "\n\n#### #{m[:title]}\n\n"    ## note: make sure to always add two newlines
+    end
+ end
+
+
+ def squish( str )
+    ## squish more than one white space to one space
+    str.gsub( /[ \r\t\n]+/, ' ' )
+ end
+
+
+ def patch_about( html )
+  # <A name=about>
+  #  <H2>About this document</H2></A> 
+  #  or
+  # <A NAME="about"><H2>About this document</H2></A>
+  #   => change to (possible?)
+  #  <H2><A name=about>About this document</A></H2>
+
+   html.sub( %r{<A [ ] name=(about|"about")> \s*
+              <H2>About [ ] this [ ] document</H2></A>
+             }ixm,
+              "<H2><A name=about>About this document</A></H2>"
+            )
+ end
+
+  # <a name="sa">Série A</a>
+  # <a name="sd">Série D</a> 
+
+  # <A name=about>
+  #  <H2>About this document</H2></A>
+  #   => change to (possible?)
+  #  <H2><A name=about>About this document</A></H2>
+  #
+  #
+  # <h4><a name="cb">Copa do Brasil</a></h4>
+
+   ## note  - for content use non-greedy to allow 
+   ##              match of tags inside content too
+
+  A_NAME_RE = %r{<A [ ]+ NAME [ ]* =
+                     (?<name>[^>]+?) 
+                  >
+                     (?<title>.+?) 
+                  </A>
+                }imx
+       
+  # <a href="#sa">Série A</a><br>
+  # 
+  #  <A href="http://www.rsssf.org/">Rec.Sport.Soccer 
+  #        Statistics Foundation</A> 
+  #  <A href="http://www.rsssfbrasil.com">RSSSF 
+  #    Brazil</A> 
+  #
+  #  and Daniel Dalence (<A
+  #   href="mailto:danielballack@terra.com.br">danielballack@terra.com.br</A>)
+
+
+  A_HREF_RE = %r{<A \s+ HREF [ ]* = 
+                    (?<href>[^>]+?)
+                  >
+                    (?<title>.+?)
+                  <\/A>
+                }imx              
+
+
+def replace_a_href( html )
+  ## remove anchors (a href)
+  #    note: heading 4 includes anchor (thus, let anchors go first)
+  #  note: <a \newline href is used for authors email - thus incl. support for newline as space
+  html.gsub( A_HREF_RE ) do |match|   ## note: use .+? non-greedy match
+    m = Regexp.last_match
+    href  = m[:href].gsub( /["']/, '' ).strip   ## remove ("" or '')
+    title = m[:title].strip   ## note: "save" caputure first; gets replaced by gsub (next regex call)
+
+
+    ## e.g.
+    ##  ‹Larsen23@gmx.de, see page mailto:Larsen23@gmx.de›
+    ##  ‹danielballack@terra.com.br, see page mailto:danielballack@terra.com.br›
+    ##  ‹zja70@aol.com, see page mailto:zja70@aol.com›)
+    if href.start_with?( 'mailto:')
+      puts " blank mailto  -  anchor (a) href >#{href}, >#{title}<"
+      '‹mailto›'   ## delete/remove email
+    else
+      puts " replace anchor (a) href >#{href}, >#{title}<"
+
+      ## convert href to xref
+      xref = if href.start_with?('#')    ## in-page ref
+              ", see §#{href[1..-1]}"
+             elsif href.start_with?( /https?:/ )            ## external page ref
+               ## skip - keep empty - why? why not? (or add url domain?)
+               ''
+             else
+               ## hack - check for some custom excludes  
+               if title.start_with?( 'Rec.Sport.Soccer' )
+                    ## skip - keep empty
+                    '' 
+               else   
+                 ## strip (ending)  .htm|html
+                 ", see page #{href.sub( /\.html?$/,'')}"
+               end
+             end
+
+      "‹#{squish(title)}#{xref}›"
+    end
+  end
+end
+
+def replace_a_name( html )
+  ##
+  ## remove (named) anchors
+  html.gsub( A_NAME_RE ) do |match|   ## note: use .+? non-greedy match
+    m = Regexp.last_match
+    name = m[:name].gsub( /["']/, '' ).strip   ## remove ("" or '')
+    title = m[:title].strip   ## note: "save" caputure first; gets replaced by gsub (next regex call)
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace anchor (a) name >#{name}<, >#{title}<    -    >#{match}<"
+  
+
+   ##
+   ## todo - report WARN if title incl. tags
+   ##    assumes text only for now - why? why not?
+   ##  add a name inside heading !!!
+   ##  do NOT add heading inside a name !!!
+
+    "#{title}  ‹§#{name}›"   ## note - use two spaces min (between title & name)
+  end
+end
+
+
+EMAIL_RE = %r{ \s*
+\(
+ [a-z][a-z0-9_]+
+   @[a-z]+(\.[a-z]+)+
+ \)
+}imx   
+
+
+def remove_emails( html )
+  ### remove converted ("blineded") mailto anchors
+  ##  note   usually inside () e.g.
+  ##    (‹mailto›) 
+  ##   plus slurp up all leading whitespace (incl. newline) - why? why not?
+  html = html.gsub( /\s*
+                      \(‹mailto›\)
+                     /xm, '' )
+  
+   ###
+   ##  remove "regular emails too e.g.
+   ##
+   ## Thanks to Marcelo Leme de Arruda (___@___.__.br),
+   ##  Ricardo FF Pontes (___@____.com), 
+   ## Santiago Reis (____@____.com.br),
+   ## Marcos Lacerda Queiroz (___@____.com.br)
+   ##  etc.
+
+  ## check for "free-standing e.g. on its own line" emails only for now
+   html = html.gsub( EMAIL_RE ) do |match|
+    puts "removing  email >#{match}<"
+    ''   
+   end
+   html
+end
+
+
+
+def html_to_txt( html )
+
+###
+#   todo: check if any tags (still) present??
+
+
+  ## cut off everything before body
+  html = html.sub( /.+?<BODY>\s*/im, '' )
+
+  ## cut off everything after body (closing)
+  html = html.sub( /<\/BODY>.*/im, '' )
+
+  html = patch_about( html )
+
+  ## remove cite
+  html = html.gsub( /<CITE>([^<]+)<\/CITE>/im ) do |_|
+    puts " remove cite >#{$1}<"
+    "#{$1}"
+  end
+
+  html = html.gsub( /\s*<HR>\s*/im ) do |match|
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace horizontal rule (hr) - >#{match}<"
+    "\n=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n"    ## check what hr to use use  - . - . - or =-=-=-= or somehting distinct?
+  end
+
+  ## replace break (br)
+  ## note: do NOT use m/multiline for now - why? why not??
+  html = html.gsub( /<BR>\s*/i ) do |match|    ## note: include (swallow) "extra" newline
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace break (br) - >#{match}<"
+    "\n"
+  end
+
+
+  
+  html = replace_a_href( html )
+  ##  note a name="about" includes more a hrefs etc.
+  #    let it go first (before a href)
+  html = replace_a_name( html )
+
+
+
+  ## replace paragrah (p)
+  html = html.gsub( /\s*<P>\s*/im ) do |match|    ## note: include (swallow) "extra" newline
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace paragraph (p) - >#{match}<"
+    "\n\n"
+  end
+  html = html.gsub( /<\/P>/i, '' )  ## replace paragraph (p) closing w/ nothing for now
+
+  ## remove i
+  html = html.gsub( /<I>([^<]+)<\/I>/im ) do |_|
+    puts " remove italic (i) >#{$1}<"
+    "#{$1}"
+  end
+
+
+  html = replace_h2( html )
+  html = replace_h4( html )
+
+
+
+
+  ## remove b   - note: might include anchors (thus, call after anchors)
+  html = html.gsub( /<B>([^<]+)<\/B>/im ) do |_|
+    puts " remove bold (b) >#{$1}<"
+    "**#{$1}**"
+  end
+
+  ## replace preformatted (pre)
+  html = html.gsub( /<PRE>|<\/PRE>/i ) do |_|
+    puts " replace preformatted (pre)"
+    ''  # replace w/ nothing for now (keep surrounding newlines)
+  end
+
+=begin
+  puts
+  puts
+  puts "html:"
+  puts html[0..2000]
+  puts "-- snip --"
+  puts html[-1000..-1]   ## print last hundred chars
+=end
+
+
+  html = remove_emails( html )
+
+
+  ## cleanup whitespaces
+  ##   todo/fix:  convert newline in space first
+  ##                and than collapse spaces etc.!!!
+  txt = String.new
+  html.each_line do |line|
+     line = line.gsub( "\t", '  ' ) # replace all tabs w/ two spaces for nwo
+     line = line.rstrip             # remove trailing whitespace (incl. newline/formfeed)
+
+     txt << line
+     txt << "\n"
+  end
+
+  txt
+end # method html_to_text
+
+
+
+###
+# more helpers
+def log( msg )
+  ## append msg to ./logs.txt  
+  ##     use ./errors.txt - why? why not?
+  File.open( './logs.txt', 'a:utf-8' ) do |f|
+    f.write( msg )
+    f.write( "\n" ) 
+  end
+end
+
+
+
+end # module PageConverter
+end # module Rsssf
+

data/lib/rsssf/download.rb

@@ -0,0 +1,151 @@
+
+module Rsssf
+
+## end_year to slug_year
+##   check if generic rule/convention in use ???
+## 2007-08: tablesd/duit08.html
+## 2008-09: tablesd/duit09.html
+## 2009-10: tablesd/duit2010.html
+## 2010-11: tablesd/duit2011.html
+## 2011-12: tablesd/duit2012.html
+
+
+    ## map country codes to table pages
+    ##   add options about (char) encoding ??? - why? why not?
+  TABLE = {
+    'eng' => ['tablese/eng{year}',   { encoding: 'Windows-1252' } ],
+    'es'  => ['tabless/span{year}',  { encoding: 'Windows-1252' } ],
+    'de'  => ['tablesd/duit{year}', { encoding: 'Windows-1252' } ],
+    'at'  => ['tableso/oost{year}', { encoding: 'Windows-1252' }  ],
+    'br'  => [
+              ->(season) {
+                  ## note: special slug/case for year/season 2000
+                  ##  see rsssf.org/tablesb/brazchamp.html
+                 if season == Season('2000') 
+                   'tablesb/braz-joao{year}'  ## use braz-joao00 - why? why not?
+                 else
+                   'tablesb/braz{year}'
+                 end
+              },  { encoding: 'Windows-1252' } ],
+  }
+
+
+  BASE_URL = "https://rsssf.org"
+
+
+  def self.table_url( code, season: )
+     url, _ = table_url_and_encoding( code, season: season )
+     url 
+  end
+
+  def self.table_url_and_encoding( code, season: )
+     season = Season( season )
+
+     table = TABLE[ code.downcase ]
+     tmpl     = table[0]
+     tmpl     = tmpl.call( season )  if tmpl.is_a?(Proc)  ## check for proc
+   
+     opts     = table[1] || {}
+     encoding = opts[:encoding]  || 'UTF-8'
+
+
+     slug =  if season.end_year < 2010   ## cut off all digits (only keep last two)s
+                 ##  convert end_year to string with leading zero
+                 '%02d' % (season.end_year % 100)  ## e.g. 00 / 01 / 99 / 98 / 11 / etc.
+              else
+                '%4d' % season.end_year
+              end
+
+     tmpl = tmpl.sub( '{year}', slug )
+     url = "#{BASE_URL}/#{tmpl}.html"
+
+     [url, encoding]
+  end
+
+
+  def self.download_table( code, season: )
+     url, encoding = table_url_and_encoding( code, season: season )
+
+     download_page( url, encoding: encoding )
+  end
+
+
+  def self.download_page( url, encoding: )
+ 
+    ## note: assume plain 7-bit ascii for now
+    ##  -- assume rsssf uses ISO_8859_15 (updated version of ISO_8859_1) 
+    ###-- does NOT use utf-8 character encoding!!!
+    response = Webget.page( url, encoding: encoding )  ## fetch (and cache) html page (via HTTP GET)
+
+    ## note: exit on get / fetch error - do NOT continue for now - why? why not?
+    exit 1   if response.status.nok?    ## e.g.  HTTP status code != 200
+  
+
+    puts "html:"
+    html =  response.text( encoding: encoding )    
+    pp html[0..400]
+    html
+  end
+end  # module Rsssf
+
+
+
+__END__
+
+1998-99: tablesd/duit99.html
+1999-00: tablesd/duit00.html      ## use 1999-2000  - why?? why not??
+2000-01: tablesd/duit01.html
+2001-02: tablesd/duit02.html
+2002-03: tablesd/duit03.html
+2003-04: tablesd/duit04.html
+2004-05: tablesd/duit05.html
+2005-06: tablesd/duit06.html
+2006-07: tablesd/duit07.html
+2007-08: tablesd/duit08.html
+2008-09: tablesd/duit09.html
+2009-10: tablesd/duit2010.html
+2010-11: tablesd/duit2011.html
+2011-12: tablesd/duit2012.html
+2012-13: tablesd/duit2013.html
+2013-14: tablesd/duit2014.html
+2014-15: tablesd/duit2015.html
+
+
+2010-11: tableso/oost2011.html
+2011-12: tableso/oost2012.html
+2012-13: tableso/oost2013.html
+2013-14: tableso/oost2014.html
+2014-15: tableso/oost2015.html
+2015-16: tableso/oost2016.html
+
+2011: tablesb/braz2011.html  !! Windows-1252
+2012: tablesb/braz2012.html  !! Windows-1252
+2013: tablesb/braz2013.html  !! Windows-1252
+2014: tablesb/braz2014.html  !! Windows-1252
+2015: tablesb/braz2015.html  !! Windows-1252
+2016: tablesb/braz2016.html  !! Windows-1252
+2017: tablesb/braz2017.html  !! Windows-1252
+2018: tablesb/braz2018.html  !! Windows-1252
+2019: tablesb/braz2019.html  !! Windows-1252
+2020: tablesb/braz2020.html  !! Windows-1252   ## 2020/21  - extended for corona
+2021: tablesb/braz2021.html  !! Windows-1252
+2022: tablesb/braz2022.html  !! Windows-1252
+2023: tablesb/braz2023.html  !! Windows-1252
+2024: tablesb/braz2024.html  !! Windows-1252
+
+2010-11: tablese/eng2011.html  !! Windows-1252
+2011-12: tablese/eng2012.html  !! Windows-1252
+2012-13: tablese/eng2013.html  !! Windows-1252
+2013-14: tablese/eng2014.html  !! Windows-1252
+2014-15: tablese/eng2015.html  !! Windows-1252
+2015-16: tablese/eng2016.html  !! Windows-1252
+2016-17: tablese/eng2017.html  !! Windows-1252
+2017-18: tablese/eng2018.html  !! Windows-1252
+2018-19: tablese/eng2019.html  !! Windows-1252
+2019-20: tablese/eng2020.html  !! Windows-1252
+2020-21: tablese/eng2021.html  !! Windows-1252
+2021-22: tablese/eng2022.html  !! Windows-1252
+2022-23: tablese/eng2023.html  !! Windows-1252
+2023-24: tablese/eng2024.html  !! Windows-1252
+
+