webget 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7cf1205972f496fc25b62a33bc47146e4395ca9d
-  data.tar.gz: 292bf57c29732fecacd7dae0579cef6cdcba2852
+  metadata.gz: 6e0feb02f55f01692b3353f5aaf15ab2201dfb02
+  data.tar.gz: da3d146f2fc6db90e2a34c9c385d7e9e9d3c272c
 SHA512:
-  metadata.gz: 91395bb6fe0cc5f15cf0ec3f58fd1c06a7f6c97c211a04985910a27dd0826f8ff3a3cf5f336f602cbb6d776169c4900be07405da8ec332f3b9b6dbcc16365e70
-  data.tar.gz: 4c834cdc685210163ce9fadadfdda4224a888886826e6a6ec17f5d07ee3cd787fa109b8da4e952b322f5e15a91087ffd740ac8f91e35d13e1608571ba0d3a590
+  metadata.gz: 49fb39362398e09eac27d929f0689a604234818f8ff6fc2422217977e6312fe46bf0273059d9310ba9b40dea62b75bc437b6e90cc36d5e39456d1927e36739ee
+  data.tar.gz: b81245bc4a7029eae5712aa94f48784240b6666d48ddc270c360980bc6f5599508ff49c7d128bafdff8419b378ab1c6d83282090a6ec10ad1a77a228fb70232a

data/Manifest.txt

@@ -4,3 +4,6 @@ README.md
 Rakefile
 lib/webget.rb
 lib/webget/version.rb
+lib/webget/webcache.rb
+lib/webget/webclient.rb
+lib/webget/webget.rb

data/README.md

@@ -2,10 +2,10 @@
 
 webget gem - yet (another) network client for world wide web (www) requests via HTTP
 
-* home  :: [github.com/rubylibs/fotos](https://github.com/rubylibs/fotos)
-* bugs  :: [github.com/rubylibs/fotos/issues](https://github.com/rubylibs/fotos/issues)
-* gem   :: [rubygems.org/gems/fotos](https://rubygems.org/gems/fotos)
-* rdoc  :: [rubydoc.info/gems/fotos](http://rubydoc.info/gems/fotos)
+* home  :: [github.com/rubycoco/webget](https://github.com/rubycoco/webget)
+* bugs  :: [github.com/rubycoco/webget/issues](https://github.com/rubycoco/webget/issues)
+* gem   :: [rubygems.org/gems/webget](https://rubygems.org/gems/webget)
+* rdoc  :: [rubydoc.info/gems/webget](http://rubydoc.info/gems/webget)
 
 
 ## Usage

data/Rakefile

@@ -5,7 +5,7 @@ Hoe.spec 'webget' do
 
   self.version = Webget::VERSION
 
-  self.summary = 'webget gem - yet (another) network client for world wide web (www) requests '
+  self.summary = 'webget gem - yet (another) network client for world wide web (www) requests'
   self.description = summary
 
   self.urls    = { home: 'https://github.com/rubycoco/fetcher' }

data/lib/webget.rb

@@ -1,7 +1,35 @@
+require 'pp'
+require 'time'
+require 'date'
+require 'fileutils'
 
-# our own code
-require 'webget/version'   # note: let version always go first
+require 'uri'
+require 'net/http'
+require 'net/https'
 
+require 'json'
+require 'yaml'
 
-# say hello
-puts Webget.banner    if defined?( $RUBYLIBS_DEBUG )
+
+
+## our own code
+require 'webget/version'   # let version go first
+require 'webget/webclient'
+require 'webget/webcache'
+require 'webget/webget'
+
+
+
+
+############
+## add convenience alias for camel case / alternate different spelling
+WebCache  = Webcache
+WebClient = Webclient
+WebGet    = Webget
+
+## use Webgo as (alias) name (keep reserver for now) - why? why not?
+WebGo    = Webget
+Webgo    = Webget
+
+
+puts Webget.banner   # say hello

data/lib/webget/version.rb

@@ -1,14 +1,16 @@
 
 class Webget
+
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 0
-  PATCH = 1
+  MINOR = 1
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version
     VERSION
   end
 
+  # version string for generator meta tag (includes ruby version)
   def self.banner
     "webget/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
   end
@@ -16,5 +18,6 @@ class Webget
   def self.root
     "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}"
   end
-end # module Webget
+
+end  # module Webget
 

data/lib/webget/webcache.rb

@@ -0,0 +1,167 @@
+
+
+module Webcache
+
+  #####
+  # copied from props gem, see Env.home
+  #    - https://github.com/rubycoco/props/blob/master/props/lib/props/env.rb
+  #   todo/fix: use original - and do NOT copy-n-paste!!! - why? why not?
+  def self.home
+    path = if( ENV['HOME'] || ENV['USERPROFILE'] )
+             ENV['HOME'] || ENV['USERPROFILE']
+           elsif( ENV['HOMEDRIVE'] && ENV['HOMEPATH'] )
+             "#{ENV['HOMEDRIVE']}#{ENV['HOMEPATH']}"
+           else
+             begin
+                File.expand_path('~')
+             rescue
+                if File::ALT_SEPARATOR
+                   'C:/'
+                else
+                   '/'
+                end
+             end
+           end
+
+    ## note: use File.expand_path to "unify" path e.g
+    ##  C:\Users\roman  becomes
+    ##  C:/Users/roman
+
+    File.expand_path( path )
+ end
+
+
+ class Configuration
+    ## root directory - todo/check: find/use a better name - why? why not?
+    def root()       @root || "#{Webcache.home}/.cache"; end
+    def root=(value) @root = value; end
+ end # class Configuration
+
+
+ ## lets you use
+ ##   Webcache.configure do |config|
+ ##      config.root = './cache'
+ ##   end
+ def self.configure() yield( config ); end
+ def self.config()    @config ||= Configuration.new;  end
+
+
+ ## add "high level" root convenience helpers
+ def self.root()       config.root; end
+ def self.root=(value) config.root = value; end
+
+
+ ### "interface" for "generic" cache storage (might be sqlite database or filesystem)
+ def self.cache() @cache ||= DiskCache.new; end
+
+ def self.record( url, response, format: 'html' )
+   cache.record( url, response, format: format );
+ end
+ def self.cached?( url ) cache.cached?( url ); end
+ class << self
+   alias_method :exist?, :cached?
+ end
+ def self.url_to_id( url ) cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
+ def self.read( url ) cache.read( url ); end
+
+
+class DiskCache
+  def cached?( url )
+    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    File.exist?( body_path )
+  end
+  alias_method :exist?, :cached?
+
+
+  def read( url )
+    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    File.open( body_path, 'r:utf-8' ) {|f| f.read }
+  end
+
+
+  ## add more save / put / etc. aliases - why? why not?
+  ##  rename to record_html - why? why not?
+  def record( url, response, format: 'html' )
+
+    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    meta_path = "#{body_path}.meta.txt"
+
+    ## make sure path exits
+    FileUtils.mkdir_p( File.dirname( body_path ) )
+
+
+    puts "[cache] saving #{body_path}..."
+
+    ## todo/check: verify content-type - why? why not?
+    if format == 'json'
+      File.open( body_path, 'w:utf-8' ) {|f| f.write( JSON.pretty_generate( response.json )) }
+    else
+      ## note - for now always assume utf8!!!!!!!!!
+      File.open( body_path, 'w:utf-8' ) {|f| f.write( response.text ) }
+    end
+
+    File.open( meta_path, 'w:utf-8' ) do |f|
+      response.headers.each do |key, value|  # iterate all response headers
+        f.write( "#{key}: #{value}" )
+        f.write( "\n" )
+      end
+    end
+  end
+
+
+
+  ### note: use file path as id for DiskCache  (is different for DbCache/SqlCache?)
+  ##    use file:// instead of disk:// - why? why not?
+  def url_to_id( str ) "disk://#{url_to_path( str )}"; end
+
+
+  ### helpers
+  def url_to_path( str )
+    ## map url to file path
+    uri = URI.parse( str )
+
+    ## note: ignore scheme (e.g. http/https)
+    ##         and  post  (e.g. 80, 8080, etc.) for now
+    ##    always downcase for now (internet domain is case insensitive)
+    host_dir = uri.host.downcase
+
+    ## "/this/is/everything?query=params"
+    ##   cut-off leading slash and
+    ##    convert query ? =
+    req_path = uri.request_uri[1..-1]
+
+
+
+    ### special "prettify" rule for weltfussball
+    ##   /eng-league-one-2019-2020/  => /eng-league-one-2019-2020.html
+    if host_dir.index( 'weltfussball.de' ) ||
+       host_dir.index( 'worldfootball.net' )
+          if req_path.end_with?( '/' )
+             req_path = "#{req_path[0..-2]}.html"
+          else
+            puts "ERROR: expected request_uri for >#{host_dir}< ending with '/'; got: >#{req_path}<"
+            exit 1
+          end
+    elsif host_dir.index( 'football-data.org' )
+      req_path = req_path.sub( 'v2/', '' )  # shorten - cut off v2/
+
+      ## flattern - make a file path - for auto-save
+      ##   change ? to -I-
+      ##   change / to ~~
+      ##   change = to ~
+      req_path = req_path.gsub( '?', '-I-' )
+                         .gsub( '/', '~~' )
+                         .gsub( '=', '~')
+
+      req_path = "#{req_path}.json"
+    else
+      ## no special rule
+    end
+
+    page_path = "#{host_dir}/#{req_path}"
+    page_path
+  end
+end # class DiskCache
+
+
+end  # module Webcache

data/lib/webget/webclient.rb

@@ -0,0 +1,85 @@
+
+class Webclient
+
+  class Response   # nested class - wrap Net::HTTP::Response
+    def initialize( response )
+      @response = response
+    end
+    def raw() @response; end
+
+
+    def text
+      # note: Net::HTTP will NOT set encoding UTF-8 etc.
+      # will be set to ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+      # thus, set/force encoding to utf-8
+      text = @response.body.to_s
+      text = text.force_encoding( Encoding::UTF_8 )
+      text
+    end
+
+    ## convenience helper; returns parsed json data
+    def json() JSON.parse( text ); end
+
+
+
+    class Headers # nested (nested) class
+      def initialize( response )
+        @response = response
+      end
+      def each( &blk )
+        @response.each_header do |key, value|  # Iterate all response headers
+          blk.call( key, value )
+        end
+      end
+    end
+    def headers() @headers ||= Headers.new( @response ); end
+
+    class Status  # nested (nested) class
+      def initialize( response )
+        @response = response
+      end
+      def code() @response.code.to_i; end
+      def ok?()  code == 200; end
+      def nok?() code != 200; end
+      def message() @response.message; end
+    end
+    def status() @status ||= Status.new( @response ); end
+  end # (nested) class Response
+
+
+def self.get( url, headers: {} )
+
+  uri = URI.parse( url )
+  http = Net::HTTP.new( uri.host, uri.port )
+
+  if uri.instance_of? URI::HTTPS
+    http.use_ssl     = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+  end
+
+  request = Net::HTTP::Get.new( uri.request_uri )
+
+  ### add (custom) headers if any
+  ##  check/todo: is there are more idiomatic way for Net::HTTP ???
+  ##   use
+  ##     request = Net::HTTP::Get.new( uri.request_uri, headers )
+  ##    why? why not?
+  ##  instead of e.g.
+  ##   request['X-Auth-Token'] = 'xxxxxxx'
+  ##   request['User-Agent']   = 'ruby'
+  ##   request['Accept']       = '*/*'
+  if headers && headers.size > 0
+    headers.each do |key,value|
+      request[ key ] = value
+    end
+  end
+
+
+  response = http.request( request )
+
+  ## note: return "unified" wrapped response
+  Response.new( response )
+end  # method self.get
+
+end  # class Webclient
+

data/lib/webget/webget.rb

@@ -0,0 +1,64 @@
+
+class Webget   # a web (go get) crawler
+
+  class Configuration  ## nested class
+
+    #######################
+    ## accessors
+    def sleep()       @sleep || 3; end     ### todo/check: use delay / wait or such?
+    def sleep=(value) @sleep = value; end
+
+  end # (nested) class Configuration
+
+  ## lets you use
+  ##   Webget.configure do |config|
+  ##      config.sleep = 10
+  ##   end
+  def self.configure() yield( config ); end
+  def self.config()    @config ||= Configuration.new;  end
+
+
+
+  def self.call( url, headers: {} )  ## assumes json format
+    puts "  sleep #{config.sleep} sec(s)..."
+    sleep( config.sleep )   ## slow down - sleep 3secs before each http request
+
+    response = Webclient.get( url, headers: headers )
+
+    if response.status.ok?  ## must be HTTP 200
+      puts "#{response.status.code} #{response.status.message}"
+      ## note: use format json for pretty printing and parse check!!!!
+      Webcache.record( url, response,
+                       format: 'json' )
+    else
+      ## todo/check - log error
+      puts "!! ERROR - #{response.status.code} #{response.status.message}:"
+      pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
+    end
+
+    ## to be done / continued
+    response
+  end  # method self.call
+
+
+  def self.page( url, headers: {} )  ## assumes html format
+    puts "  sleep #{config.sleep} sec(s)..."
+    sleep( config.sleep )   ## slow down - sleep 3secs before each http request
+
+    response = Webclient.get( url, headers: headers )
+
+    if response.status.ok?  ## must be HTTP 200
+      puts "#{response.status.code} #{response.status.message}"
+      Webcache.record( url, response )   ## assumes format: html (default)
+    else
+      ## todo/check - log error
+      puts "!! ERROR - #{response.status.code} #{response.status.message}:"
+      pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
+    end
+
+    ## to be done / continued
+    response
+  end  # method self.page
+
+end  # class Webget
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webget
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Gerald Bauer
@@ -44,7 +44,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.22'
-description: 'webget gem - yet (another) network client for world wide web (www) requests '
+description: webget gem - yet (another) network client for world wide web (www) requests
 email: ruby-talk@ruby-lang.org
 executables: []
 extensions: []
@@ -59,6 +59,9 @@ files:
 - Rakefile
 - lib/webget.rb
 - lib/webget/version.rb
+- lib/webget/webcache.rb
+- lib/webget/webclient.rb
+- lib/webget/webget.rb
 homepage: https://github.com/rubycoco/fetcher
 licenses:
 - Public Domain