webget-mirror 0.0.1

data/lib/webget/mirror/website.rb

@@ -0,0 +1,204 @@
+
+class Webget
+  class Mirror
+
+
+class Website
+  def host
+     raise ArgumentError, "website.base_url not set"   if @base_url.nil?
+     @base_url.host
+  end
+
+  def base_url=( url )
+      @base_url = URI( url )   ## note URI() same as URI.parse()
+  end
+  def base_url
+      raise ArgumentError, "website.base_url not set"   if @base_url.nil?
+      @base_url.to_s     ## note - returns string NOT uri object!!!
+  end
+
+
+
+  ## array of hash (table) records e.g.
+  ##    page,         encoding
+  ##    /index.html,  windows-1252
+  def start_pages=( config )
+    @start_pages = config
+  end
+  def start_pages
+      raise ArgumentError, "website.start_pages not set"   if @start_pages.nil?
+      @start_pages
+  end
+
+
+  def start_pages_path
+      ## collect/build path for all start pages
+      ## e.g.  path: ['/curdom.html',
+      ##               '/curtour.html',
+      ##               '/histdom.html',
+      ##               '/intclub.html',
+      ##               '/intland.html']
+      path = []
+      start_pages.each do |config|
+          path << config['page']
+      end
+      path
+  end
+  alias_method :start_path, :start_pages_path
+
+
+  ###
+  ## give preference on schedule next page batch
+  ##          if path matches sql like term/str
+  ##           e.g.  '/table%'
+  ##   => Page.where( 'path LIKE ?', '/table%' )
+  ##
+    ## note - sql (like) wildcard rules/syntax:
+       ##      %: Matches zero or more characters
+       ##      _: Matches exactly one character
+
+
+  def boost_pages_path_like=(str)  @boost = str; end
+  def boost_pages_path_like?() defined?( @boost ); end
+  def boost_pages_path_like()  defined?( @boost ) ? @boost : nil; end
+
+  alias_method :boost_path_like=, :boost_pages_path_like=
+  alias_method :boost_path_like?, :boost_pages_path_like?
+  alias_method :boost_path_like,  :boost_pages_path_like
+
+
+
+
+
+  def default_page_encoding=( encoding )
+       @default_page_encoding = encoding
+  end
+  def _default_page_encoding
+       defined?( @default_page_encoding ) ?  @default_page_encoding : 'UTF-8'
+  end
+
+  ##
+  ##   use hash with custom (individual) defaults e.g.
+  ##    Hash.new { |h,key| h[key] = 'windows-1252'  }
+  ##
+  def page_encodings=( encodings )  ## hash (path => encoding)
+      @page_encodings = encodings
+  end
+
+  ## default page encoding (lookup by path);
+  ##    change to windows-1256 if needed
+  def page_encoding( path )
+     if defined?( @page_encodings )
+        @page_encodings[ path ]
+     else
+       _default_page_encoding
+     end
+  end
+
+
+
+  def errata_edits=(edits)  @errata_edits = edits; end
+
+  ### quick fix html w/ search & replace
+  ##    default: do nothing
+  def errata?()   defined?( @errata_edits ); end
+  def errata( html, url: )
+       if defined?( @errata_edits )
+
+         ## lookup edits by path e.g. /tablesp/poland-satrip77.html
+         ##                       or  /miscellaneous/torre-madrid.html
+
+         page_url = URI( url )
+
+         edits = @errata_edits[page_url.path]
+
+         ## note - for now always use gsub (not sub)
+         ##   maybe add option later
+         if edits
+           edits.each do |search,replace|
+                         html = html.gsub( search, replace )
+                      end
+         end
+         html
+       else   ## pass along as is (1:1)
+         html
+       end
+  end
+
+
+  def autofix_href=( m )
+       raise ArgumentError,
+               "proc expected for autofix_href=; got #{m} #{m.class.name}"   unless m.is_a?( Proc )
+
+       @autofix_href = m
+  end
+  def autofix_href()  defined?( @autofix_href ) ? @autofix_href : nil; end
+
+
+
+  ### kick-off mirror (pages) operation/run
+  def mirror
+
+
+    ## add seed/start pages
+    start_pages.each do |config|
+      path     = config['page']
+      encoding = config['encoding']
+      encoding = page_encoding( path )   if config['encoding'].nil? || config['encoding'].empty?
+
+      page_rec = MirrorDb::Model::Page.find_or_create_by!( path: path ) do |rec|
+                      ## note - block only called on create (NOT find!!)
+                      puts "  add page #{rec.path} (cached: false) to mirror.db"
+
+                      rec.encoding = encoding
+                      rec.cached   = false
+                end
+      pp page_rec
+
+
+
+      ## fix-fix-fix use Mirror.new( self )
+      ##         pass in site config on new!!!
+      mirror = Mirror.new
+      mirror._mirror_pages( site: self )
+    end
+  end
+end  # class Website
+end  ## class Mirror
+end  ## class Webget
+
+
+
+
+__END__
+
+
+=begin
+module Mirror
+  class Configuration
+     def host() @base_url.host; end
+
+     def base_url
+        raise ArgumentError, "config.base_url not set"   if @base_url.nil?
+        @base_url
+     end
+     def base_url=( url )
+        @base_url = URI( url )   ## note URI() same as URI.parse()
+     end
+  end # class Configuration
+
+
+ ## lets you use
+ ##   Webcache.configure do |config|
+ ##      config.root = './cache'
+ ##   end
+ def self.configure() yield( config ); end
+ def self.config()    @config ||= Configuration.new;  end
+
+
+ ## add "high level" root convenience helpers
+ ##   use delegate helper - why? why not?
+ ## def self.host()       config.host; end
+ ## def self.host=(value) config.host = value; end
+end   # module Mirror
+=end

data/lib/webget/mirror.rb

@@ -0,0 +1,55 @@
+$LOAD_PATH.unshift( '/sports/rubycocos/webclient/webclient/lib' )
+$LOAD_PATH.unshift( '/sports/rubycocos/webclient/webget/lib' )
+
+
+require 'cocos'
+require 'webget'           ## incl. webget, webcache, webclient, etc.
+require 'nokogiri'
+
+require 'active_record'   ## todo: add sqlite3? etc.
+
+
+### our own
+require_relative 'mirror/version'     ## version first
+
+require_relative 'mirror/database/schema'
+require_relative 'mirror/database/models'
+require_relative 'mirror/database/open'
+
+
+
+
+require_relative 'mirror/download_page'
+require_relative 'mirror/find_links'  ## find_links helper 'n' more
+require_relative 'mirror/collect_page_info'
+
+
+require_relative 'mirror/mirror'
+
+
+require_relative 'mirror/utils'
+require_relative 'mirror/website'
+
+
+
+class Webget
+  class Mirror
+
+## auto log errors  (append to logs.txt)
+def log( msg )
+   ## append msg to ./logs.txt
+   ##     use ./errors.txt - why? why not?
+   File.open( './mirror_logs.txt', 'a:utf-8' ) do |f|
+     f.write( msg )
+     f.write( "\n" )
+   end
+end
+
+end  ## class Mirror
+end  ## class Webget
+
+
+
+
+
+puts Webget::Mirror.banner    ## say hello

data/lib/webget-mirror.rb

@@ -0,0 +1,2 @@
+## alternative single-file for  "webget/mirror"
+require_relative 'webget/mirror'

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification
+name: webget-mirror
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Gerald Bauer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-09-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: webget
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+description: webget-mirror gem - addon to webget to mirror websites
+email: gerald.bauer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+- Rakefile
+- lib/webget-mirror.rb
+- lib/webget/mirror.rb
+- lib/webget/mirror/collect_page_info.rb
+- lib/webget/mirror/database/models.rb
+- lib/webget/mirror/database/open.rb
+- lib/webget/mirror/database/schema.rb
+- lib/webget/mirror/download_page.rb
+- lib/webget/mirror/find_links.rb
+- lib/webget/mirror/mirror.rb
+- lib/webget/mirror/utils.rb
+- lib/webget/mirror/version.rb
+- lib/webget/mirror/website.rb
+homepage: https://github.com/rubycocos/webclient
+licenses:
+- Public Domain
+metadata: {}
+post_install_message:
+rdoc_options:
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: webget-mirror gem - addon to webget to mirror websites
+test_files: []