webget-mirror 0.0.1

data/lib/webget/mirror/find_links.rb

@@ -0,0 +1,222 @@
+
+
+class Webget
+  class Mirror
+
+
+   ## double assert
+      ## assert - double check
+          ## make sure url.path does NOT start with // or
+          ##                              /// !!
+         ##  and does NOT end_with /
+         ##    pages
+
+def _broken_path?( path )
+      path.start_with?( '//' ) ||
+      path.end_with?( '/' ) ||
+     !path.start_with?( '/' )  ## note - MUST start with single slash (/)
+end
+
+
+
+## get all links
+##   ignore anchor links and
+##     split into internal and external
+def _find_links( site:,
+                 doc:,
+                 url:,
+                 verbose: true )
+
+
+       ## note - base_url is  the doc(ument) url
+       ##                            e.g.  https://rsssf.org/curtour.html
+       base_url = URI( url )
+
+
+           if _broken_path?( base_url.path )
+              puts "!! normalized base_url.path expected  - got:"
+              pp url
+              pp base_url
+              exit 1
+           end
+
+    ##
+    ## note: Array#compact removes all nil values from an array.
+    ##    if no href in a - nokigiri return nil
+    ##
+    ##     might still incl. empty string ("") - remove too - why? why not?
+
+    ##
+    ##  fix - change to css('a[href]') or such ??
+    ###     document.css("a[href]").each do |a|
+    ## links = doc.css('a').map { |a| a['href'] }.compact
+
+
+    links = doc.css( 'a[href]' ).map do |a|
+                    ## strip leading & trailing spaces e.g.
+                    ##   "http://www.danskfodbold.dk "
+                    ##    is invalid url!!!
+                    a['href'].strip
+                end.reject do |href|
+                    # skip
+                    #  - empty strings,
+                    #  - page anchors
+                    href.empty? || href.start_with?('#') ||
+
+                    ##  skip mailto links/javascript snippets
+                    href.match?( /\A(?:mailto|javascript)/i ) ||
+
+                     ## also skip broken mailto links
+                     ##   that is, missing mailto
+                     ##  e.g.
+                     href.include?( '@' )
+                end
+
+
+    ## split into internal & external
+    ## make links absolute
+    ##   ignore anchor links (see above)
+
+    pages     = []
+    externals = []
+
+    links.each do |href|
+
+        ##
+        ## auto-fix ("site-wide") known quirks:
+        href = site.autofix_href.call( href )    if  site.autofix_href.is_a?( Proc )
+
+
+                      page_url = nil
+                      begin
+
+                        ## special case
+                        ##  check for protocol-relative  //  e.g. //hello.html
+                        ##    NOT handled by URI
+                        ##       URI makes hello.html into host !!!
+                        ##                host is hello.html and path is nil
+                        ##        only works properly with triple ///
+                        ##             e.g. ///hello.html
+                        ##              now host is nil, and path is /hello.html
+
+      ##   URI.join(URI("https://example.com/page.html"), "//cdn.example.com/file.js")
+      ##  # => #<URI::HTTPS https://cdn.example.com/file.js>  ✓ Works!
+      ##
+      ##   But with just the string:
+      ##    URI("//cdn.example.com/file.js")
+      ##     Parses incorrectly—no scheme, treats cdn.example.com as host  !!!!!
+
+
+       ##  The browser breaks down //path/page.html like this:
+       ##  - Protocol: Inherited from the current page (e.g., https:).
+       ##  - Domain (Authority): path
+       ##  - File Path: /page.html
+       ##
+       ## If your website is hosted on https://example.com and
+       ## a user clicks <a href="//path/page.html">, the browser will try
+       ## to navigate to https://path/page.html.
+       ## Unless you own a domain name that is literally just path,
+       ##  this will result in a "Site cannot be reached" error.
+       ###
+       ###  "legacy" protocol relative is "//://" !!!!
+       ##
+       ##   move notes from here to dedicated notes page!!
+
+
+
+                        if href.start_with?("//")
+                           puts "!!! debug break on href starting with //:"
+                           pp  href
+                           pp  url
+                           pp  base_url
+                           exit 1
+                        end
+
+
+
+                        ## check if href is absolute?
+                        href_url = URI( href )
+
+                        ## assume already absolute
+                        if href_url.scheme && href_url.host
+                          page_url = href_url
+                        else
+                          ## try to make absolute (relative to base_url)
+                          page_url = URI.join(base_url, href_url)
+                        end
+
+                      rescue => ex
+                         ## skip bad urls and log
+
+                         msg = "bad url in #{base_url.path}:\n#{href}\nex:#{ex}\n"
+
+                         ## note - only report in verbose mode (fresh download or such)!!!
+                         if verbose
+                           log( msg )
+                           puts "!! " + msg
+                         end
+
+                         next
+                      end
+
+                      ###
+                      ##  fix-fix-fix
+                      ##    check for  optional www too
+                      ##          assume same for now ??
+                      ##    or better add to autofix
+                      ##            if www.rsssf.org  change to  rsssf.org
+
+                      if page_url.host == site.host    ## e.g. 'rsssf.org'
+                          if page_url.path == base_url.path
+                                 puts "   anchor  #{href}  =>  #{page_url.fragment}"     if verbose
+                          else
+                               puts "   internal page  #{href}  =>  #{page_url.path}"     if verbose
+
+                               ## note - for internal pages
+                               ##  for now no SUPPORT for query
+                               ##    e.g. foo=1&bar=2
+                               if page_url.query
+                                   ## change to ValueError or such - why? why not?
+                                   ## raise ArgumentError, "query in internal page links not yet supported, sorry - got #{page_url}"
+                                   msg = "query in internal page links not yet supported, sorry - got #{page_url}"
+                                   puts "!! WARN - #{msg}"
+                                   log( msg )
+                                   next
+                               end
+
+        if _broken_path?( page_url.path )
+            puts "!! normalized page_url.path expected - got:"
+            pp page_url.path
+            pp page_url
+            puts "base_url:"
+            pp url
+            pp base_url
+            exit 1
+          end
+
+                               pages << page_url.path
+                          end
+                      else
+                         puts "!! external  #{href}  =>  #{page_url}"      if verbose
+                         externals << page_url.to_s
+                      end
+                    end
+
+     ## make uniq
+     pages     = pages.uniq
+     externals = externals.uniq
+
+      if verbose
+      puts "   #{pages.size} internal & #{externals.size} external link(s) found in #{base_url.path}:"
+
+
+    pp pages
+    pp externals
+      end
+
+    [pages, externals]
+end
+
+
+end  ## class Mirror
+end  ## class Webget

data/lib/webget/mirror/mirror.rb

@@ -0,0 +1,246 @@
+
+## use recursive_download_page or such - why? why not?
+##   or add recursive flag
+
+=begin
+visited: 100 (downloaded: 98) -  35586 page(s) indexed (9158 cached, 26428 missing)
+    [98/26526 -  0.00%]  2:14 mins -  1.38 secs/page, estimate: 608:53 mins
+=end
+
+
+class Webget
+  class Mirror
+
+##
+##  use limit for batch - why? why not?
+##     start of / try a batch of a hundred
+def _mirror_pages( site:,
+                   force: false,
+                   batch: 1000 )
+
+    visited    = 0
+    downloaded = 0
+
+    time_start = Time.now
+
+    loop do
+
+       ## (i)  prioritize main pages (e.g. use start_pages_path)
+       ## /curdom.html
+       ## /curtour.html
+       ## /histdom.html
+       ## /intclub.html
+       ## /intland.html
+      page_recs =  MirrorDb::Model::Page.where( cached: false,
+                                                path:   site.start_pages_path
+                                               ).limit( batch )
+
+
+      ##  (ii)   prefer pages   (e.g. use boost_pages_path_like)
+      ##  starting with /tables,/tables[a-z]/
+      if page_recs.size == 0 && site.boost_pages_path_like?
+        page_recs =  MirrorDb::Model::Page.where( cached: false ).
+                                           where( 'path LIKE ?',
+                                                  site.boost_pages_path_like ).limit( batch )
+      end
+
+      ##  (iii)  retry "unconstrained"  if nothing found matching  (i & ii)
+      if page_recs.size == 0
+        page_recs =  MirrorDb::Model::Page.where( cached: false ).limit( batch )
+      end
+
+
+      ### no more pages - done - break out of loop and say goodbye
+      break   if page_recs.size == 0
+
+
+
+
+       page_recs.each_with_index do |page_rec,i|
+
+        ##
+        ##  fix-fix-fix  - change to mime type - why? why not?
+        ##           allow pages with no extensions!!!
+
+         ### special case for non .html/.htm pages (e.g. .pdf others too??)
+         ##    do NOT download / mirror / cache for now
+         if page_rec.not_html?
+            page_rec.update!( cached: true )
+            next
+         end
+
+
+         ## note - on download (not if cached)
+         ##        encoding
+         ##           might be get changed
+         ##        ALWAYS use updated encoding!!
+
+         ##
+         ## note - workaround for windows
+         ##     on windows File.exist? (and Webcache.cached?)
+         ##          is case-insensitive
+         ##    e.g. /USAdave/ is the same as /usadave/
+         ##
+         ##   as a workaround ALWAYS hardcode 404
+         ##    for /USAdave/    to get (and record) 404  (and not CACHE HITS!!)
+         ##   e.g. try https://rsssf.org/USAdave/cncc.html  => 404 (NOT FOUND)
+         ##            https://rsssf.org/usadave/cncc.html  => 200 (OK)
+
+
+         ##  note - url e.g. https://rsssf.org
+         ##         path MUST start with /  e.g.  /curtour.html
+         ##  resulting in   https://rsssf.org/curtour.html
+
+         url = site.base_url+page_rec.path
+
+        ## if %r{/USAdave/}.match?(page_rec.path)
+        ##                                 ['', {status: 404}]
+
+
+
+  ## check if not in cache
+  ##   note - use force == true  to always (force) download
+
+          html, response_meta = _download_page( url,
+                                                encoding: page_rec.encoding,
+                                                force:    force  )
+
+          if response_meta
+              downloaded += 1
+              puts " ---  " + fmt_time_diff( time_start,  count: downloaded )
+
+              ###
+              ## special case
+              ##  check for 404 NOT FOUND
+              if response_meta[:http_status] == 404
+                      page_rec.update!( http_status: 404,
+                                        cached:      true )
+
+                next   ### note - skip further processing on 404 (no links etc.)!!
+              end
+          end
+
+
+
+          html = site.errata( html, url: url )    if site.errata?
+
+
+
+
+         ## Standard HTML4-style parsing (default)
+         ## doc = Nokogiri::HTML(malformed_html)
+         ##  -or-
+         ## More robust HTML5 parsing
+         ##doc = Nokogiri::HTML5(malformed_html)
+
+           doc = Nokogiri::HTML( html )
+
+
+           ## get (page meta info)
+           ##   title, tabs (count), html_doctype, html_charset
+            page_info = _collect_page_info( doc, html: html )
+
+
+
+          ##  note - if response meta data present than fresh download (not cached)
+          ##  cached  = response_meta ? false : true
+
+          ## turn on verbose mode only if page downloaded (not on cache hit)
+           verbose = response_meta ? true : false
+          ## verbose = true
+
+           internals, _ = _find_links( site: site,
+                                       doc: doc,
+                                       url: url,
+                                       verbose: verbose
+                                     )
+
+
+            ## add links to db
+            internals.each do |path|
+               internal_rec = MirrorDb::Model::Page.find_or_create_by!(
+                                                            path: path ) do |rec|
+                                    puts "     add linked page #{rec.path}"
+
+                                    rec.encoding = site.page_encoding( rec.path )
+                                    rec.cached   = false
+                                 end
+
+               ## puts "  add link from #{page_rec.path} to #{internal_rec.path} to mirror.db"
+               ###  note allow - find (may happen after "crash" or interrupt)
+               link_rec = MirrorDb::Model::Link.find_or_create_by!(
+                                                         from_page_id: page_rec.id,
+                                                         to_page_id:   internal_rec.id )
+            end
+
+            puts "  [#{i+1}/#{page_recs.size}] update page #{page_rec.path} w/ #{internals.size} page(s) linked - >#{page_info[:title] || 'n/a'}<"
+
+
+            ###
+            ##  note - remove cached (flag) and replace with http_status => nil|200|404|etc?
+            ##         that is, cached = false  => nil
+            ##                  cached = true   =>  200|404|etc - why? why not??
+            attribs = {
+                cached: true
+            }
+            ## add (optional) page_info attribus
+            more_attribs = {
+              title:        page_info[:title], ## note - might be missing (nil) in some pages
+              html_doctype: page_info[:html_doctype],
+              html_charset: page_info[:html_charset],
+              tabs:         page_info[:tabs]
+            }
+            attribs = attribs.merge( more_attribs )
+
+
+            ## check for encoding when fresh download (via response meta data)
+            if response_meta
+               more_attribs = {
+                  encoding:        response_meta[:encoding] ? response_meta[:encoding].downcase : nil,
+                  encoding_source: response_meta[:encoding_source],   ## bom|html|http|user|fallback
+                  encoding_valid:  response_meta[:encoding_valid],    ## nil|true|false
+
+                   ascii7bit:         response_meta[:ascii7bit],      ## nil|true|false
+                   ## note - convert 10 - 212=>8, 233=>2  (use only first total count; no details)
+                   chars_8bit:        response_meta[:chars_8bit] ? response_meta[:chars_8bit].to_i(10) : nil,
+                   utf8_replace:      response_meta[:utf8_replace],
+
+                   http_content_type:     response_meta[:http_content_type],
+                   http_content_length:   response_meta[:http_content_length],
+                   http_status:           response_meta[:http_status]
+                }
+               attribs = attribs.merge( more_attribs )
+            end
+
+
+            page_rec.update!( **attribs )
+
+
+
+            visited += 1
+
+           if visited % 100 == 0
+              puts "\n visited: #{visited} (downloaded: #{downloaded}) - " +
+                 " #{MirrorDb::Model::Page.count} page(s) indexed " +
+                 "(#{MirrorDb::Model::Page.cached.count} cached, " +
+                 "#{MirrorDb::Model::Page.not_cached.count} missing)"
+
+             puts "  " + fmt_time_diff( time_start,  step: downloaded,
+                                                      count: downloaded+MirrorDb::Model::Page.not_cached.count )
+
+
+           end
+       end
+    end
+
+
+
+            puts "\n visited: #{visited} (downloaded: #{downloaded}) - " +
+                 " #{MirrorDb::Model::Page.count} page(s) indexed " +
+                 "(#{MirrorDb::Model::Page.cached.count} cached, " +
+                 "#{MirrorDb::Model::Page.not_cached.count} missing)"
+end
+
+
+end  ## class Mirror
+end  ## class Webget

data/lib/webget/mirror/utils.rb

@@ -0,0 +1,32 @@
+
+
+class Webget
+  class Mirror
+
+
+
+def fmt_time_diff( time_start, time_end=Time.now, count:, step: nil )
+   time_diff  = time_end - time_start
+   buf = String.new
+
+     if count == 0 || step == 0
+       buf +=  "  %d:%02d mins" % [time_diff/60, time_diff%60]
+     elsif step
+       buf +=  "  [#{step}/#{count} - %5.2f%%]" % [step*100/count]
+
+       buf +=  "  %d:%02d mins" % [time_diff/60, time_diff%60]
+       buf +=  " - %5.2f secs/page" % [time_diff/step]
+
+       time_estimate = (time_diff/step) * count
+       buf +=  ", estimate: %d:%02d mins" % [time_estimate/60, time_estimate%60]
+    else
+      buf +=  "  %d:%02d mins" % [time_diff/60, time_diff%60]
+      buf +=  " - %5.2f secs/page  (#{count} pages)" % [time_diff/count]
+   end
+
+   buf
+end
+
+
+end  ## class Mirror
+end  ## class Webget

data/lib/webget/mirror/version.rb

@@ -0,0 +1,22 @@
+
+class Webget
+class Mirror
+   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
+   MINOR = 0
+   PATCH = 1
+   VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  # version string for generator meta tag (includes ruby version)
+  def self.banner
+    "webget-mirror/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}] in (#{root})"
+  end
+
+  def self.root
+    File.expand_path( File.dirname(File.dirname(File.dirname(File.dirname(__FILE__)))) )
+  end
+end  # class Mirror
+end  # class Webget