webget-mirror 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 22d6bef3214b26fff9ccc700e43479887b29888529dd51b33ad67b2506e9c1ed
+  data.tar.gz: 65bf372aaacb2cf77e0743d7315a3ad6c82b54827c0c6035b56d1851b8ee8458
+SHA512:
+  metadata.gz: 99fc9d25ab2b876130498eccc8d97482fe144e156d037fc37e6b6a7811b5a62c58a735307e6c8341466d67c21a2b3bc09307f7e41a0a8416eb8c7fc35ff3b471
+  data.tar.gz: d1addaa18484d4e8b13e19b606f9119a568ff44e1f2a3a8a36431e3153de33c161aaa84f9b226db16d00c07659c6d2164af70c49c19ac187700c916d835dc42d

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+### 0.0.1 / 2026-09-09
+
+* Everything is new. First release.

data/Manifest.txt

@@ -0,0 +1,16 @@
+CHANGELOG.md
+Manifest.txt
+README.md
+Rakefile
+lib/webget-mirror.rb
+lib/webget/mirror.rb
+lib/webget/mirror/collect_page_info.rb
+lib/webget/mirror/database/models.rb
+lib/webget/mirror/database/open.rb
+lib/webget/mirror/database/schema.rb
+lib/webget/mirror/download_page.rb
+lib/webget/mirror/find_links.rb
+lib/webget/mirror/mirror.rb
+lib/webget/mirror/utils.rb
+lib/webget/mirror/version.rb
+lib/webget/mirror/website.rb

data/README.md

@@ -0,0 +1,19 @@
+# webget-mirror
+
+webget-mirror gem - addon to webget to mirror websites
+
+* home  :: [github.com/rubycocos/webclient](https://github.com/rubycocos/webclient)
+* bugs  :: [github.com/rubycocos/webclient/issues](https://github.com/rubycocos/webclient/issues)
+* gem   :: [rubygems.org/gems/webget-mirror](https://rubygems.org/gems/webget-mirror)
+* rdoc  :: [rubydoc.info/gems/webget-mirror](http://rubydoc.info/gems/webget-mirror)
+
+
+## Usage
+
+TBD
+
+
+## License
+
+The `webget-mirror` scripts are dedicated to the public domain.
+Use  as you please with no restrictions whatsoever.

data/Rakefile

@@ -0,0 +1,34 @@
+require 'hoe'
+require './lib/webget/mirror/version.rb'
+
+Hoe.spec 'webget-mirror' do
+
+  self.version = Webget::Mirror::VERSION
+
+  self.summary = 'webget-mirror gem - addon to webget to mirror websites'
+  self.description = summary
+
+  self.urls    = { home: 'https://github.com/rubycocos/webclient' }
+
+  self.author  = 'Gerald Bauer'
+  self.email   = 'gerald.bauer@gmail.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'CHANGELOG.md'
+
+  self.extra_deps = [
+    ['webget', ],
+    ['nokogiri'],
+    ['activerecord'],
+    ['sqlite3'],
+  ]
+
+
+  self.licenses = ['Public Domain']
+
+  self.spec_extras = {
+   required_ruby_version: '>= 3.1.0'
+  }
+
+end

data/lib/webget/mirror/collect_page_info.rb

@@ -0,0 +1,107 @@
+
+
+class Webget
+  class Mirror
+
+
+###
+# fix-fix-fix
+#    make regex more "generic"
+#
+# add real-world samples here
+
+=begin
+
+rsssf.org/tableso/oost2014.html:
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-2">
+rsssf.org/tablesn/nz-intres.html:
+   <meta http-equiv="content-type" content="text/html; charset=UTF-16LE">
+rsssf.org/tablesa/argchamp.html:
+   <meta http-equiv="Content-Type" content="text/html; charset=UTFs-8">
+   => typo  - UTFs-8 !!!
+rsssf.org/tables/2002full.html:
+   <META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+rsssf.org/miscellaneous/zwed-coach-triv.html:
+   <meta http-equiv="Content-Type" content="text/html; charset=windows-1251">
+rsssf.org/tablesr/roem68.html:
+   <META http-equiv=Content-Type content="text/html; charset=windows-1250">
+
+=end
+
+
+
+HTML_CHARSET_RE = %r{
+   <meta [ ]+
+       [^<>]*?        ## note - use non-greedy (shortest) match
+  \bcharset
+        [ ]*=[ ]*
+          ['"]?       ## optional opening quote
+        (?<charset>[a-z0-9_-]+)
+}ix
+
+
+HTML_DOCTYPE_RE = %r{
+   <!DOCTYPE [ ]+
+        (?<doctype> [^<>]+?)  ## note - use non-greedy (shortest) match
+                               ## do NOT allow opening/closing brackets for now
+                               ##  ever possible? double check
+            [ ]*
+   >
+}ix
+
+
+
+def _collect_page_info( doc, html: )
+
+           ## use collect_page_stat( doc: )
+           ##   or   collect_page_info  ( pass in nokogiri doc !!)
+           ##     PageInfo   (or Page::Info), PageStat
+           ##       - title
+           ##       - html_doctype
+           ##       - html_charset
+           ##       - tabs
+           ##       ...
+
+           ### try to find page title
+           ##    not - title might be missing (nil)!!
+             title_el =  doc.at_css('title')
+             title =  title_el ? title_el.text.strip :  nil
+
+    ##
+             ##  note - use "plain-old" regex
+             ##     to get "raw" doctype/charset  from html source
+             ##
+             ## record doctype
+             ##  e.g
+             ##  <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+             ##  <!DOCTYPE HTML>
+             ## and html charset (inside meta)
+             ##  e.g.
+             ## <meta http-equiv="Content-Type" content="text/html; charset=Windows-1252">
+             ## <meta charset="Windows-1252">
+
+             ###
+             ## note -   limit search to 1024 ( or allow double 2048)
+
+             html_doctype =  (m=HTML_DOCTYPE_RE.match( html[0,1024] )) ? m[:doctype] : nil
+             html_charset =  (m=HTML_CHARSET_RE.match( html[0,1024] )) ? m[:charset] : nil
+
+
+             ## check for tabs  - make it nil if no tabs found otherwise use count
+             tabs = html.scan( "\t" )
+             tabs =  tabs.size == 0 ? nil : tabs.size
+
+
+   meta = {
+     title:        title,
+     html_doctype: html_doctype,
+     html_charset: html_charset,
+     tabs:         tabs,
+   }
+
+   meta
+end
+
+
+end  ## class Mirror
+end  ## class Webget

data/lib/webget/mirror/database/models.rb

@@ -0,0 +1,123 @@
+
+##
+# note - use a sqlite database for caching pages and (internal) links
+##            use via activerecord machinery / object-relational mapper
+
+
+module MirrorDb
+module Model
+
+
+class Page < ActiveRecord::Base
+   has_many  :outgoing_links,  class_name: 'Link',
+                               foreign_key: 'from_page_id',
+                               :dependent  => :delete_all  ## :destroy
+
+   ## use outgoing_pages or linked_pages?
+   has_many  :linked_pages,  :through => :outgoing_links,
+                             :source  => :to_page
+
+   ## backlink (incoming)
+   has_many  :incoming_links, class_name: 'Link',
+                              foreign_key: 'to_page_id',
+                               :dependent  => :delete_all  ## :destroy
+
+   ##  use incoming_pages or backlink_pages?
+   has_many  :backlink_pages, :through => :incoming_links,
+                              :source  => :from_page
+
+
+   def outgoing_paths() linked_pages.pluck(:path); end
+   def incoming_paths() backlink_pages.pluck(:path); end
+
+
+   ## find a better name for not cached (was missing)? why? why not?
+   ##   cached a.k.a. downloaded to local cache
+   scope  :cached,     -> { where( cached: true ) }
+   scope  :not_cached, -> { where( cached: false ) }
+
+   ## 404 not_found
+   scope  :not_found,  -> { where( http_status: 404 ) }
+   ##
+   ## add scope  :ok  for 200  - why ? why not?
+
+
+   ## for extname (file extensions)
+   ##   note - .html auto incl .htm !!
+   scope  :html,  -> { where( extname: ['.html', '.htm']) }
+   scope  :pdf,   -> { where( extname: '.pdf') }
+
+   def html?()      extname == '.html' || extname == '.htm';  end
+   def not_html?()  !html?();  end
+   def pdf?()       extname == '.pdf'; end
+
+
+   def not_found?()  http_status == 404; end
+   def not_cached?()  !cached?(); end
+
+
+    ### note - path incl. leading slash e.g. /curtour.html
+    ##  def url()  "#{Mirror.config.base_url}#{path}"; end
+
+
+
+
+## tip - Use before_validation instead if needed:
+##  If your callback modifies attributes that need to be validated,
+## use before_validation instead of before_create.
+##  before_create runs after validation passes
+
+   ###  note - use callback to autofill basename,extname, dirname from path
+   before_validation :autofill
+
+   ## double check added path
+   validate :assert_path
+
+private
+   def autofill
+      self.basename = File.basename( path, File.extname( path ))   if basename.nil?
+      self.extname  = File.extname( path )                         if extname.nil?
+      self.dirname  = File.dirname( path )                         if dirname.nil?
+
+      ###
+      ##  note -  always downcase extname for now - why? why not?
+      ##    possibly .HTM or .HTML (or even .Html or such)
+      ##
+      ##  maybe latter autofill format or such - why? why not?
+      self.extname = extname.downcase       if extname
+   end
+
+   def assert_path
+         ## assert - double check
+          ## make sure url.path does NOT start with // or
+          ##                              /// !!
+         ##  and does NOT end_with /
+         ##
+         ##                page_rec.path.include?( %r{/{2,}} ) ||
+         ##   fix  http.//  typos!!!
+         ##      page_rec.path.match?( %r{\.{2,}} )
+         ##   fix ..sources typos ...
+         ##    pages
+           if  path.start_with?( '//' ) ||
+               path.end_with?( '/' )  ||
+              !path.start_with?( '/' )  ## note - MUST start with single slash (/)
+                errors.add(:path, "broken; starts with // or ends with /")
+           end
+
+   end
+end # class Page
+
+
+
+
+
+class Link < ActiveRecord::Base
+   belongs_to  :from_page, class_name: 'Page',
+                           foreign_key: 'from_page_id'
+   belongs_to  :to_page,   class_name: 'Page',
+                           foreign_key: 'to_page_id'
+end # class Link
+
+
+end   # module Model
+end  # module MirrorDb

data/lib/webget/mirror/database/open.rb

@@ -0,0 +1,63 @@
+
+
+module MirrorDb
+    def self.open( path='./mirror.db' )
+
+      ### reuse connect here !!!
+      ###   why? why not?
+
+      config = {
+          adapter:  'sqlite3',
+          database: path,
+      }
+
+      ActiveRecord::Base.establish_connection( config )
+      # ActiveRecord::Base.logger = Logger.new( STDOUT )
+
+        ## try to speed up sqlite
+        ##   see http://www.sqlite.org/pragma.html
+        con = ActiveRecord::Base.connection
+        con.execute( 'PRAGMA synchronous=OFF;' )
+        con.execute( 'PRAGMA journal_mode=OFF;' )
+        con.execute( 'PRAGMA temp_store=MEMORY;' )
+
+      ##########################
+      ### auto_migrate
+      unless Model::Page.table_exists?
+          CreateDb.new.up
+      end
+    end  # method open
+end
+
+
+
+__END__
+
+
+=begin
+    def self.open_readonly( path='./mirror.db' )
+
+       ### raise ArgumentError, "sqlite db #{path} not found"  if !File.exist?( path )
+
+
+      config = {
+          adapter:  'sqlite3',
+          database: path,
+          readonly: true,   ## try readonly prop!!!
+      }
+
+      ActiveRecord::Base.establish_connection( config )
+      # ActiveRecord::Base.logger = Logger.new( STDOUT )
+
+        ## try to speed up sqlite
+        ##   see http://www.sqlite.org/pragma.html
+        con = ActiveRecord::Base.connection
+
+        ## add for read-only - why? why not?
+       # con.execute( 'PRAGMA query_only=ON;' )
+
+       # con.execute( 'PRAGMA synchronous=OFF;' )
+       # con.execute( 'PRAGMA journal_mode=OFF;' )
+       # con.execute( 'PRAGMA temp_store=MEMORY;' )
+    end
+=end

data/lib/webget/mirror/database/schema.rb

@@ -0,0 +1,67 @@
+module MirrorDb
+class CreateDb
+def up
+  ActiveRecord::Schema.define do
+
+####
+# pages tables
+create_table :pages do |t|
+   t.string :path, null: false
+
+   ##  split/break up path  - maybe make it later virtual columns - why? why not?
+   ## add basename, dirname, extname  - why? why not?
+   t.string :basename, null: false
+   t.string :dirname,  null: false
+   t.string :extname,  null: false    ##  note - empty string if no extname?
+                                      ##    if present starts with dot (.) e.g.
+                                      ##          .html|.htm, .pdf, etc.
+                                      ##   keep dot why? why not?
+
+   t.string :title          ##   html <title></title>
+   t.date   :updated        ## iso date e.g. (2026-04-29) via web page source
+
+   ################
+   ### add (charset) encoding stuff
+   t.string  :encoding           ## "upstream" text encoding
+                                 ##   all pages ALWAYS converted to utf-8
+   t.string  :encoding_source   ## e.g. bom|http|html|user|fallback
+
+   t.boolean :encoding_valid  ## uses  String#encoding_valid?
+   t.boolean :ascii7bit       ## uses  String#ascii_only?  check if all chars are ascii 7bit (utf8-compatible) ??
+   t.integer :chars_8bit       ## count of 8bit (126-255) chars - nil|0|1|2|etc.
+   t.integer :utf8_replace     ## count invalid/replace chars in utf8 - nil|0|1|2
+   t.integer :tabs         ## count tabs/tabstops in html source (use tab or tabs ??)
+
+
+   t.string  :html_doctype
+   t.string  :html_charset
+
+   t.string  :http_content_type     ## http content-type   header
+   t.integer :http_content_length   ## http content-length header
+   t.integer :http_status           ## e.g. 200, 404  - make mandatory - why? why not?
+
+   ## or use download or date (fetched) or such??
+   t.boolean :cached,   default: false
+
+  # t.timestamps  ## (auto)add - why? why not?
+  #  do NOT use; save space for now - auto-generated db is read-only
+end
+add_index :pages, :path, unique: true
+
+
+## join table  - no need for own ids
+create_table :links, id: false do |t|
+   t.integer  :from_page_id,  null: false
+   t.integer  :to_page_id,    null: false
+end
+add_index :links, [:from_page_id, :to_page_id], unique: true
+add_index :links, :from_page_id
+add_index :links, :to_page_id
+
+##
+## add errors or log or such - why? why not?
+##
+  end  # Schema.define
+end # method up
+end # class CreateDb
+end # module MirrorDb

data/lib/webget/mirror/download_page.rb

@@ -0,0 +1,83 @@
+
+
+class Webget
+  class Mirror
+
+
+def _download_page( url,
+                    encoding: nil,
+                    force: false )
+
+##
+## note - on windows cached will be CASE-INSENSITIVE
+##       e.g. usadave and USAdave will match
+##    make sure match is CASE-SENSITIVE!!!
+
+          if force == false && Webcache.cached?( url )
+              puts "   CACHE HIT - #{url}"
+              html  = Webcache.read( url )
+
+              return [html,nil]
+          end
+
+
+        puts "==> download #{url} (encoding: #{encoding})..."
+
+
+    ## note: assume plain 7-bit ascii for now
+    ##  -- assume rsssf uses ISO_8859_15 (updated version of ISO_8859_1)
+    ###-- does NOT use utf-8 character encoding!!!
+    response = Webget.page( url, encoding: encoding )  ## fetch (and cache) html page (via HTTP GET)
+
+    ## note: exit on get / fetch error - do NOT continue for now - why? why not?
+    ## note -    allow 404 not found to pass through
+
+
+    ## note - status.code is an integer number (not a string!!)
+    if response.status.code == 404
+
+        meta = {
+           http_status:     response.status.code,
+        }
+
+        ["404 NOT FOUND",meta]
+
+    elsif response.status.code == 200
+      puts "html:"
+      html =  response.text
+      pp html[0..200]
+
+      ## note - use "hacky" undocument internal response._text_encoding
+      ##                   to get "upstream" encoding used from convert to utf-8
+      ##                         unicode boms may override user supplied encoding!!!
+      ##  or change upstream
+      ##   and    use/add response.text_with_encoding( ) - why? why not?
+      ##   yes, upstream now uses
+      ##    text( encoding: _encoding_user )!!!
+
+
+      meta = {
+          encoding:         response._text_encoding,
+          encoding_source:  response._text_encoding_source,  ## bom|http|html|user|fallback
+          encoding_valid:   response._text_encoding_valid,
+
+          ascii7bit:         response._text_ascii_only,
+          chars_8bit:        response._text_8bit,
+          utf8_replace:      response._text_utf8_replace,
+
+          http_content_type:     response.content_type,
+          http_content_length:   response.content_length,
+          http_status:           response.status.code,
+      }
+
+        [html,meta]
+
+    else
+       puts "unexpected http status (code) - #{response.status}"
+       exit 1
+    end
+end
+
+
+end ## class Mirror
+end ## class Webget