rsssf 0.1.0 → 0.3.0

data/lib/rsssf/convert/html_to_txt/remove_emails.rb

@@ -0,0 +1,43 @@
+module Rsssf
+class PageConverter
+
+
+EMAIL_RE = %r{ \s*
+\(
+ [a-z][a-z0-9_]+
+   @[a-z]+(\.[a-z]+)+
+ \)
+}imx   
+
+
+def remove_emails( html )
+  ### remove converted ("blinded") mailto anchors
+  ##  note   usually inside () e.g.
+  ##    (‹mailto›) 
+  ##   plus slurp up all leading whitespace (incl. newline) - why? why not?
+  html = html.gsub( /\s*
+                      \(‹mailto›\)
+                     /xm, '' )
+  
+   ###
+   ##  remove "regular emails too e.g.
+   ##
+   ## Thanks to Marcelo Leme de Arruda (___@___.__.br),
+   ##  Ricardo FF Pontes (___@____.com), 
+   ## Santiago Reis (____@____.com.br),
+   ## Marcos Lacerda Queiroz (___@____.com.br)
+   ##  etc.
+
+  ## check for "free-standing e.g. on its own line" emails only for now
+   html = html.gsub( EMAIL_RE ) do |match|
+    puts "removing  email >#{match}<"
+    ''   
+   end
+   html
+end
+
+
+
+end # module PageConverter
+end # module Rsssf
+

data/lib/rsssf/convert/html_to_txt/replace_a_href.rb

@@ -0,0 +1,85 @@
+module Rsssf
+class PageConverter
+
+ # <a href="#sa">Série A</a><br>
+  # 
+  #  <A href="http://www.rsssf.org/">Rec.Sport.Soccer 
+  #        Statistics Foundation</A> 
+  #  <A href="http://www.rsssfbrasil.com">RSSSF 
+  #    Brazil</A> 
+  #
+  #  and Daniel Dalence (<A
+  #   href="mailto:danielballack@terra.com.br">danielballack@terra.com.br</A>)
+  ##
+  ##
+  ##  empty
+  ##  <a>Primer Descenso – First Relegation</a>
+
+  A_HREF_RE = %r{<A 
+                    (?: 
+                       \s+ HREF [ ]* = 
+                          (?<href>[^>]+?)
+                    )?
+                  >
+                    (?<title>.+?)
+                  <\/A>
+                }imx              
+
+
+def replace_a_href( html )
+  ## remove anchors (a href)
+  #    note: heading 4 includes anchor (thus, let anchors go first)
+  #  note: <a \newline href is used for authors email - thus incl. support for newline as space
+  html.gsub( A_HREF_RE ) do |match|   ## note: use .+? non-greedy match
+    m = Regexp.last_match
+    captures = m.named_captures
+    href  = if m['href']
+               m['href'].gsub( /["']/, '' ).strip   ## remove ("" or '')
+            else
+               nil
+            end
+    title = m['title'].strip   ## note: "save" caputure first; gets replaced by gsub (next regex call)
+
+
+    if href.nil?
+       ## report error - <a>hello</a> is useless
+       puts " replace anchor w/ missing (!!) href (a) >#{title}<"
+      "‹#{squish(title)}›"
+  
+    ## e.g.
+    ##  ‹Larsen23@gmx.de, see page mailto:Larsen23@gmx.de›
+    ##  ‹danielballack@terra.com.br, see page mailto:danielballack@terra.com.br›
+    ##  ‹zja70@aol.com, see page mailto:zja70@aol.com›)
+  
+    elsif href.start_with?( 'mailto:')
+      puts " blank mailto  -  anchor (a) href >#{href}, >#{title}<"
+      '‹mailto›'   ## delete/remove email
+    else
+      puts " replace anchor (a) href >#{href}, >#{title}<"
+
+      ## convert href to xref
+      xref = if href.start_with?('#')    ## in-page ref
+              ", see §#{href[1..-1]}"
+             elsif href.start_with?( /https?:/ )            ## external page ref
+               ## skip - keep empty - why? why not? (or add url domain?)
+               ''
+             else
+               ## hack - check for some custom excludes  
+               if title.start_with?( 'Rec.Sport.Soccer' )
+                    ## skip - keep empty
+                    '' 
+               else   
+                 ## strip (ending)  .htm|html
+                 ", see page #{href.sub( /\.html?$/,'')}"
+               end
+             end
+
+      "‹#{squish(title)}#{xref}›"
+    end
+  end
+end
+
+
+
+end # module PageConverter
+end # module Rsssf

data/lib/rsssf/convert/html_to_txt/replace_a_name.rb

@@ -0,0 +1,87 @@
+module Rsssf
+class PageConverter
+
+
+  # <a name="sa">Série A</a>
+  # <a name="sd">Série D</a> 
+
+  # <A name=about>
+  #  <H2>About this document</H2></A>
+  #   => change to (possible?)
+  #  <H2><A name=about>About this document</A></H2>
+  #
+  #
+  # <h4><a name="cb">Copa do Brasil</a></h4>
+
+   ## note  - for content use non-greedy to allow 
+   ##              match of tags inside content too
+
+  A_NAME_OLD_RE = %r{<A [ ]+ NAME [ ]* =
+                     (?<name>[^>]+?) 
+                  >
+                     (?<title>.+?) 
+                  </A>
+                }imx
+
+
+  A_NAME_RE = %r{<A [ ]+ NAME [ ]* =
+                     (?<name>[^>]+?) 
+                  >
+                }imx
+
+                
+def replace_a_name_old( html )
+  ##
+  ## remove (named) anchors
+  html.gsub( A_NAME_RE ) do |match|   ## note: use .+? non-greedy match
+    m = Regexp.last_match
+    name = m[:name].gsub( /["']/, '' ).strip   ## remove ("" or '')
+    title = m[:title].strip   ## note: "save" caputure first; gets replaced by gsub (next regex call)
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace anchor (a) name >#{name}<, >#{title}<    -    >#{match}<"
+  
+
+   ##
+   ## todo - report WARN if title incl. tags
+   ##    assumes text only for now - why? why not?
+   ##  add a name inside heading !!!
+   ##  do NOT add heading inside a name !!!
+
+    "#{title}  ‹§#{name}›"   ## note - use two spaces min (between title & name)
+  end
+end
+
+
+
+def replace_a_name( html )
+
+  ## note - allows <a name=""> without closing </a>
+  ##    <a name="semi"><H2>Semifinals</H2>
+  ##   always put anchor on its own line for now
+
+  ##
+  ## remove (named) anchors
+  html.gsub( A_NAME_RE ) do |match|   ## note: use .+? non-greedy match
+    m = Regexp.last_match
+    
+    name = m[:name].gsub( /["']/, '' ).strip   ## remove ("" or '')
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace anchor (a) name >#{name}<    -    >#{match}<"
+  
+   ##
+   ## todo - report WARN if title incl. tags
+   ##    assumes text only for now - why? why not?
+   ##  add a name inside heading !!!
+   ##  do NOT add heading inside a name !!!
+
+    "‹§#{name}›"   ## note - use two spaces min (between title & name)
+  end
+end
+
+
+
+
+end # module PageConverter
+end # module Rsssf
+
+

data/lib/rsssf/convert/html_to_txt/replace_heading.rb

@@ -0,0 +1,76 @@
+module Rsssf
+class PageConverter
+
+
+  ## note - for h1,h2,h3,h4,h5,h6
+  ##     use a backref(erence) e.g. \1
+  ##
+  ## note - include leading and trailing spaces (incl. newlines) !!!
+  ##
+  ## note  - for content inside use non-greedy to allow 
+  ##              match of tags inside content too
+  HEADING_RE = %r{ \s*
+                     <H(?<level>[1-6])>
+                       (?<title> .+?)
+                     </H\k<level>>
+                     \s*
+                   }imx
+
+
+
+  ###
+  ##  note - MUST be a one a single line (see make heading for more)   
+  ##           e.g.    "<h#{tag}>#{text}</h#{tag}>"              
+  BOLD_OR_UNDERLINE_LINE_HEADING_RE = %r{^
+                          [ ]*
+                     <H (?<tag> [BU]) >
+                       (?<title> .+?)
+                     </H \k<tag> >
+                         [ ]*
+                          $
+                   }ix
+  
+                 
+  def replace_heading( html )
+     html = html.gsub( HEADING_RE ) do |_|
+        m = Regexp.last_match
+
+        level = m[:level].to_i(10) 
+        title = m[:title]
+
+        puts " replace heading #{level} (h#{level}) >#{title}<"
+
+        ## note: make sure to always add two newlines before and after
+        "\n\n#{'='*level} #{title}\n\n"    
+        
+     end
+
+     html = html.gsub( BOLD_OR_UNDERLINE_LINE_HEADING_RE ) do |_|
+        m = Regexp.last_match
+
+        tag = m[:tag].downcase 
+        title = m[:title]
+
+        ## use heading 5 for b and heading 6 for underline for now
+        ##   maybe later change to custom  ==_ or ==* or such
+        ##     to mark the heading (sourced via bold/underscore) ???
+        level =  if tag == 'b'
+                      5
+                 elsif tag == 'u'
+                      6 
+                 else
+                     raise ArgumentError, "b(old)|u(underscore) tag expected; got #{tag}"
+                 end
+
+        puts " replace #{tag}-heading #{level} (h#{level}) >#{title}<"
+
+        ## note: do NOT add any newlines before and after
+        "#{'='*level} #{title}"    
+     end
+   
+    html
+  end
+
+
+end # module PageConverter
+end # module Rsssf

data/lib/rsssf/convert/html_to_txt/replace_hr.rb

@@ -0,0 +1,25 @@
+module Rsssf
+class PageConverter
+
+
+
+
+HR_LINE_ASCII = "\n\n=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n\n"
+
+def replace_hr( html )
+
+  html = html.gsub( /\s*<HR>\s*/im ) do |match|
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace horizontal rule (hr) - >#{match}<"
+    HR_LINE_ASCII    ## check what hr to use use  - . - . - or =-=-=-= or somehting distinct?
+  end
+ 
+  html
+end
+
+end # module PageConverter
+end # module Rsssf
+
+
+
+

data/lib/rsssf/convert/html_to_txt.rb

@@ -0,0 +1,247 @@
+module Rsssf
+class PageConverter
+
+
+
+
+
+
+ def squish( str )
+    ## squish more than one white space to one space
+    str.gsub( /[ \r\t\n]+/, ' ' )
+ end
+
+
+
+
+def html_to_txt( html, url: )
+
+###
+#   todo: check if any tags (still) present??
+
+
+  ## cut off everything before body
+  ##   
+  ## note - might incl. attributes e.g.
+  ## <body bgcolor="yellow">
+
+  ## record / track (important) edits - sub(stitutions) etc.
+  edits = []
+
+
+  html = html.sub( /.+?
+                      <BODY [^>]*? >
+                      \s*
+                   /xim, 
+                   '' )
+                  
+  ## special case i)   no <body> - cut-off head if present
+  ## cut off everything before <head/>
+  ##   used in braz93.html, braz98.html
+  html = html.sub( /.+?
+                     <\/HEAD>
+                       \s*
+                   /xim, '' )    
+
+  ## special case ii) no <body>, no </head> 
+  ## cut off everything before <head/>
+  ##   used in braz93.html, braz98.html
+  html = html.sub( /.+?
+                     <HEAD\/>
+                       \s*
+                   /xim, '' )    
+
+
+
+
+  ## cut off everything after body (closing)
+  html = html.sub( /<\/BODY>.*/im, '' )
+  
+  ## special case
+  ## cut off everything after </html> (closing)
+  ##   used in braz93.html, braz98.html
+  html = html.sub( /<\/HTML>.*/im, '' )
+
+
+
+
+  ## quick fix
+  ## <title>World Cup 1950 qualifications</title>
+  ## <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-2">
+  
+  ## remove title and meta
+  html = html.sub( /<TITLE>.*?<\/TITLE>/i, '' )
+  html = html.sub( /<META .*?>/i, '' )
+
+
+  ## (auto-)fix known types / errors
+  ##  todo - pass in/along filename/url too - why? why not?
+  html = errata_html( html )
+
+
+
+  ##
+  ##   change ^<b><a name ...></a></b>$  or    <hb>  - heading "bold" - might be h5
+  ##          ^<u><a name ...></a></u>$    to  <hu>  - heading "underscore" - might be h6
+  html, more_edits = make_heading( html )
+  edits += more_edits 
+
+
+
+
+
+
+  ## remove cite
+  html = html.gsub( /<CITE>([^<]+)<\/CITE>/im ) do |_|
+    puts " remove cite >#{$1}<"
+    "#{$1}"
+  end
+
+
+  html = replace_hr( html )
+
+
+
+
+  ## replace break (br)
+  ## note: do NOT use m/multiline for now - why? why not??
+  html = html.gsub( /<BR>\s*/i ) do |match|    ## note: include (swallow) "extra" newline
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace break (br) - >#{match}<"
+    "\n"
+  end
+
+
+
+  
+  html = replace_a_name( html )
+
+  html = replace_a_href( html )
+
+  ## quickfix  remove trailing </a> left possibly by a_name
+  html = html.gsub( /<\/A>/i, '' )
+
+
+
+  ## replace paragrah (p)
+  html = html.gsub( /\s*<P>\s*/im ) do |match|    ## note: include (swallow) "extra" newline
+    match = match.gsub( "\n", '$$' )  ## make newlines visible for debugging
+    puts " replace paragraph (p) - >#{match}<"
+    "\n\n"
+  end
+  html = html.gsub( /<\/P>/i, '' )  ## replace paragraph (p) closing w/ nothing for now
+
+
+
+ 
+  html = replace_heading( html )
+
+
+  ## remove i(talics)
+  ##    use non-greedy match as default? e.g. .*? - why? why not?
+  ##    or greedy w/  ([^<]+)
+  html = html.gsub( /<I>(.*?)<\/I>/im ) do |_|
+    puts " remove italic (i) >#{$1}<"
+    "#{$1}"
+  end
+
+  html = html.gsub( /<U>(.*?)<\/U>/im ) do |_|
+    puts " remove underline (u) >#{$1}<"
+    "#{$1}"
+  end
+
+  ## remove b   - note: might include anchors (thus, call after anchors)
+  ###   use non-greedy match as default? e.g. .*? - why? why not?
+  ## was - "**#{$1}**"
+  html = html.gsub( /<B>(.*?)<\/B>/im ) do |_|
+    puts " remove bold (b) >#{$1}<"
+    "#{$1}"  
+  end
+
+  ## <strong></strong>
+  html = html.gsub( /<STRONG>(.*?)<\/STRONG>/im ) do |_|
+    puts " remove strong (strong) >#{$1}<"
+    "#{$1}"  
+  end
+
+
+
+  ## replace preformatted (pre)
+  html = html.gsub( /<PRE>|<\/PRE>/i ) do |match|
+    puts " replace preformatted (pre)"
+
+      ## note - replace preformatted blocks
+      ##           with comments 
+      ##  was:
+      ##  ''  # replace w/ nothing for now (keep surrounding newlines)
+      
+     if match.downcase == '<pre>'
+         '<!-- start pre -->'
+     else
+         '<!-- end pre -->'
+     end
+  end
+
+
+=begin
+  puts
+  puts
+  puts "html:"
+  puts html[0..2000]
+  puts "-- snip --"
+  puts html[-1000..-1]   ## print last hundred chars
+=end
+
+
+  html = remove_emails( html )
+
+
+  html = beautify_anchors( html )
+
+
+  ## check for html tags
+  ##  left
+  ##  use scan instead of 
+  html.gsub( /<
+                \/?
+                [A-Z]+ [^>]*
+              > 
+             /xim ) do |match|
+
+          if ['<menu>', '<ul>', '<li>',
+               '</menu>', '</ul>', '</li>'].include?(match.downcase) 
+               ## do nothing
+          else 
+                  msg = "found unprocessed html tag #{match} in >#{url}<"
+                  puts "*** WARN - #{msg}"
+                  log( msg )  ## log too (see log.txt)
+          end 
+          match
+    end
+
+
+  ##
+  ## todo/fix
+  ##    move up-front - kind of preprocessing (not post) - why? why not?
+
+  ## cleanup whitespaces
+  ##   todo/fix:  convert newline in space first
+  ##                and than collapse spaces etc.!!!
+  txt = String.new
+  html.each_line do |line|
+     line = line.gsub( "\t", '  ' ) # replace all tabs w/ two spaces for nwo
+     line = line.rstrip             # remove trailing whitespace (incl. newline/formfeed)
+
+     txt << line
+     txt << "\n"
+  end
+
+  txt = errata_txt( txt )
+
+  [txt, edits]
+end # method html_to_text
+
+
+
+end # module PageConverter
+end # module Rsssf

data/lib/rsssf/download.rb

@@ -0,0 +1,20 @@
+
+module Rsssf
+
+  def self.download_page( url, encoding: )
+
+    ## note: assume plain 7-bit ascii for now
+    ##  -- assume rsssf uses ISO_8859_15 (updated version of ISO_8859_1)
+    ###-- does NOT use utf-8 character encoding!!!
+    response = Webget.page( url, encoding: encoding )  ## fetch (and cache) html page (via HTTP GET)
+
+    ## note: exit on get / fetch error - do NOT continue for now - why? why not?
+    exit 1   if response.status.nok?    ## e.g.  HTTP status code != 200
+
+
+    puts "html:"
+    html =  response.text( encoding: encoding )
+    pp html[0..400]
+    html
+  end
+end  # module Rsssf