webget 0.1.1 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f30b181118ed0b78e2617246be67187afbc82b06
-  data.tar.gz: a70be1694115592004e896f2b762db231896fdf2
+SHA256:
+  metadata.gz: e171b577175334da7546a23003e44796c19da96b37c35a21947e5f025772cf19
+  data.tar.gz: c3a9a39e443edef2b702a15096cf5f5135a4aaccfc9fe1ef9fe8b0e9e9ce9296
 SHA512:
-  metadata.gz: 2279af9b0b84949ef8e175acdda2e91cdf2d41bda3a6048bfd43e1759731fe3f92f8f9f4008137017659644d4d013257bb551fb7d823c3217f1808470df3a8cf
-  data.tar.gz: 5551d7778d4c8e2664a34aa6f873189be48fdf64681e4a6b84eb9e73f11ec5109ccf5e5b6a0cdba9ee6bcea7f35bc9a1337723d5fe9ff1b3c8ca74eacaed7963
+  metadata.gz: bf0c524fd19e2444a98df172ccf217d1c9a6201849dc3d947e2885229294271fc865df00cb329c2840a149a72e819facf224c07b2b555ef161874af2cb03c3e1
+  data.tar.gz: 44532a070b15e02cba5a861de875d25e16484d05dc1c0b8793b4e0874deaa94a464dafe8be047bc36284f9f1c9358760c6738f41e9828e2a15e0845e8f09c9d2

data/Manifest.txt

@@ -5,5 +5,4 @@ Rakefile
 lib/webget.rb
 lib/webget/version.rb
 lib/webget/webcache.rb
-lib/webget/webclient.rb
 lib/webget/webget.rb

data/README.md

@@ -1,11 +1,12 @@
 # webget
 
-webget gem - yet (another) network client for world wide web (www) requests via HTTP
+webget gem - a web (go get) crawler incl. web cache
 
-* home  :: [github.com/rubycoco/fetcher](https://github.com/rubycoco/fetcher)
-* bugs  :: [github.com/rubycoco/fetcher/issues](https://github.com/rubycoco/fetcher/issues)
+* home  :: [github.com/rubycoco/webclient](https://github.com/rubycoco/webclient)
+* bugs  :: [github.com/rubycoco/webclient/issues](https://github.com/rubycoco/webclient/issues)
 * gem   :: [rubygems.org/gems/webget](https://rubygems.org/gems/webget)
 * rdoc  :: [rubydoc.info/gems/webget](http://rubydoc.info/gems/webget)
+* forum :: [groups.google.com/group/wwwmake](https://groups.google.com/group/wwwmake)
 
 
 ## Usage

data/Rakefile

@@ -5,10 +5,10 @@ Hoe.spec 'webget' do
 
   self.version = Webget::VERSION
 
-  self.summary = 'webget gem - yet (another) network client for world wide web (www) requests'
+  self.summary = 'webget gem - a web (go get) crawler incl. web cache'
   self.description = summary
 
-  self.urls    = { home: 'https://github.com/rubycoco/fetcher' }
+  self.urls    = { home: 'https://github.com/rubycoco/webclient' }
 
   self.author  = 'Gerald Bauer'
   self.email   = 'ruby-talk@ruby-lang.org'
@@ -17,7 +17,10 @@ Hoe.spec 'webget' do
   self.readme_file  = 'README.md'
   self.history_file = 'CHANGELOG.md'
 
-  self.extra_deps = []
+  self.extra_deps = [
+    ['webclient', '>= 0.2.0'],
+    ['csvreader', '>= 1.2.4'],
+  ]
 
   self.licenses = ['Public Domain']
 

data/lib/webget.rb

@@ -1,20 +1,11 @@
-require 'pp'
-require 'time'
-require 'date'
-require 'fileutils'
-
-require 'uri'
-require 'net/http'
-require 'net/https'
-
-require 'json'
-require 'yaml'
+require 'webclient'
 
+## more (our own) 3rd party libs
+require 'csvreader'
 
 
 ## our own code
 require 'webget/version'   # let version go first
-require 'webget/webclient'
 require 'webget/webcache'
 require 'webget/webget'
 
@@ -24,7 +15,6 @@ require 'webget/webget'
 ############
 ## add convenience alias for camel case / alternate different spelling
 WebCache  = Webcache
-WebClient = Webclient
 WebGet    = Webget
 
 ## use Webgo as (alias) name (keep reserver for now) - why? why not?

data/lib/webget/version.rb

@@ -2,8 +2,8 @@
 class Webget
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
-  MINOR = 1
-  PATCH = 1
+  MINOR = 2
+  PATCH = 4
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/webget/webcache.rb

@@ -54,16 +54,24 @@ module Webcache
  ### "interface" for "generic" cache storage (might be sqlite database or filesystem)
  def self.cache() @cache ||= DiskCache.new; end
 
- def self.record( url, response, format: 'html' )
-   cache.record( url, response, format: format );
+ def self.record( url, response,
+                   path: nil,
+                   encoding: 'UTF-8',
+                   format: 'html' )
+   cache.record( url, response,
+                   path: path,
+                   encoding: encoding,
+                   format: format );
  end
  def self.cached?( url ) cache.cached?( url ); end
  class << self
    alias_method :exist?, :cached?
  end
- def self.url_to_id( url ) cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
+ def self.url_to_id( url )  cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
  def self.read( url )       cache.read( url );      end
  def self.read_json( url )  cache.read_json( url ); end
+ def self.read_csv( url )   cache.read_csv( url );  end
+
 
 
 class DiskCache
@@ -86,13 +94,22 @@ class DiskCache
     data
   end
 
+  def read_csv( url )
+    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    txt = File.open( body_path, 'r:utf-8' ) {|f| f.read }
+    data = CsvHash.parse( txt )
+    data
+  end
 
 
   ## add more save / put / etc. aliases - why? why not?
   ##  rename to record_html - why? why not?
-  def record( url, response, format: 'html' )
+  def record( url, response,
+              path: nil,
+              encoding: 'UTF-8',
+              format: 'html' )
 
-    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    body_path = "#{Webcache.root}/#{url_to_path( url, path: path )}"
     meta_path = "#{body_path}.meta.txt"
 
     ## make sure path exits
@@ -102,14 +119,23 @@ class DiskCache
     puts "[cache] saving #{body_path}..."
 
     ## todo/check: verify content-type - why? why not?
+    ## note - for now respone.text always assume (converted) to utf8!!!!!!!!!
     if format == 'json'
       File.open( body_path, 'w:utf-8' ) {|f| f.write( JSON.pretty_generate( response.json )) }
-    else
-      ## note - for now always assume utf8!!!!!!!!!
-      File.open( body_path, 'w:utf-8' ) {|f| f.write( response.text ) }
+    elsif format == 'csv'
+      ## fix: newlines - always use "unix" style" - why? why not?
+      ## fix:  use :newline => :universal option? translates to univeral "\n"
+      text = response.text( encoding: encoding ).gsub( "\r\n", "\n" )
+      File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
+    else   ## html or txt
+      text = response.text( encoding: encoding )
+      File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
     end
 
+
     File.open( meta_path, 'w:utf-8' ) do |f|
+      ## todo/check:
+      ##  do headers also need to converted (like text) if encoding is NOT utf-8 ???
       response.headers.each do |key, value|  # iterate all response headers
         f.write( "#{key}: #{value}" )
         f.write( "\n" )
@@ -125,7 +151,7 @@ class DiskCache
 
 
   ### helpers
-  def url_to_path( str )
+  def url_to_path( str, path: nil )
     ## map url to file path
     uri = URI.parse( str )
 
@@ -134,10 +160,14 @@ class DiskCache
     ##    always downcase for now (internet domain is case insensitive)
     host_dir = uri.host.downcase
 
-    ## "/this/is/everything?query=params"
-    ##   cut-off leading slash and
-    ##    convert query ? =
-    req_path = uri.request_uri[1..-1]
+    req_path = if path   ## use "custom" (file)path for cache storage if passed in
+                 path
+               else
+                ## "/this/is/everything?query=params"
+                ##   cut-off leading slash and
+                ##    convert query ? =
+                 uri.request_uri[1..-1]
+               end
 
 
 
@@ -151,6 +181,25 @@ class DiskCache
             puts "ERROR: expected request_uri for >#{host_dir}< ending with '/'; got: >#{req_path}<"
             exit 1
           end
+    elsif host_dir.index( 'tipp3.at' )
+      req_path = req_path.sub( '.jsp', '' )  # shorten - cut off .jsp extension
+
+      ##   change ? to -I-
+      ##   change = to ~
+      ##   Example:
+      ##   sportwetten/classicresults.jsp?oddsetProgramID=888
+      ##     =>
+      ##   sportwetten/classicresults-I-oddsetProgramID~888
+      req_path = req_path.gsub( '?', '-I-' )
+                         .gsub( '=', '~')
+
+      req_path = "#{req_path}.html"
+    elsif host_dir.index( 'fbref.com' )
+      req_path = req_path.sub( 'en/', '' )      # shorten - cut off en/
+      req_path = "#{req_path}.html"             # auto-add html extension
+    elsif host_dir.index( 'football-data.co.uk' )
+      req_path = req_path.sub( 'mmz4281/', '' )  # shorten - cut off mmz4281/
+      req_path = req_path.sub( 'new/', '' )      # shorten - cut off new/
     elsif host_dir.index( 'football-data.org' )
       req_path = req_path.sub( 'v2/', '' )  # shorten - cut off v2/
 

data/lib/webget/webget.rb

@@ -19,7 +19,7 @@ class Webget   # a web (go get) crawler
 
 
 
-  def self.call( url, headers: {} )  ## assumes json format
+  def self.call( url, headers: {} )  ## assumes json format (note - encoding always utf-8 by definition! - double check?)
     puts "  sleep #{config.sleep} sec(s)..."
     sleep( config.sleep )   ## slow down - sleep 3secs before each http request
 
@@ -40,8 +40,8 @@ class Webget   # a web (go get) crawler
     response
   end  # method self.call
 
-
-  def self.page( url, headers: {} )  ## assumes html format
+  ## todo/check: rename encoding to html/http-like charset - why? why not?
+  def self.page( url, encoding: 'UTF-8', headers: {} )  ## assumes html format
     puts "  sleep #{config.sleep} sec(s)..."
     sleep( config.sleep )   ## slow down - sleep 3secs before each http request
 
@@ -49,7 +49,8 @@ class Webget   # a web (go get) crawler
 
     if response.status.ok?  ## must be HTTP 200
       puts "#{response.status.code} #{response.status.message}"
-      Webcache.record( url, response )   ## assumes format: html (default)
+      Webcache.record( url, response,
+                       encoding: encoding  )   ## assumes format: html (default)
     else
       ## todo/check - log error
       puts "!! ERROR - #{response.status.code} #{response.status.message}:"
@@ -60,5 +61,54 @@ class Webget   # a web (go get) crawler
     response
   end  # method self.page
 
+
+  def self.text( url, path: nil, headers: {} )  ## assumes txt format
+    puts "  sleep #{config.sleep} sec(s)..."
+    sleep( config.sleep )   ## slow down - sleep 3secs before each http request
+
+    response = Webclient.get( url, headers: headers )
+
+    if response.status.ok?  ## must be HTTP 200
+      puts "#{response.status.code} #{response.status.message}"
+      ## note: like json assumes always utf-8 encoding for now !!!
+      Webcache.record( url, response,
+                       path: path,   ## optional "custom" (file)path for saving in cache
+                       format: 'txt' )
+    else
+      ## todo/check - log error
+      puts "!! ERROR - #{response.status.code} #{response.status.message}:"
+      pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
+    end
+
+    ## to be done / continued
+    response
+  end  # method self.text
+
+
+
+  ## todo/check: rename to csv or file or records or - why? why not?
+  ## todo/check: rename encoding to html/http-like charset - why? why not?
+  def self.dataset( url, encoding: 'UTF-8', headers: {} )  ## assumes csv format
+    puts "  sleep #{config.sleep} sec(s)..."
+    sleep( config.sleep )   ## slow down - sleep 3secs before each http request
+
+    response = Webclient.get( url, headers: headers )
+
+    if response.status.ok?  ## must be HTTP 200
+      puts "#{response.status.code} #{response.status.message}"
+      Webcache.record( url, response,
+                       encoding: encoding,
+                       format:   'csv' )    ## pass along csv format - why? why not?
+    else
+      ## todo/check - log error
+      puts "!! ERROR - #{response.status.code} #{response.status.message}:"
+      pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
+    end
+
+    ## to be done / continued
+    response
+  end  # method self.dataset
+
+
 end  # class Webget
 

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: webget
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.4
 platform: ruby
 authors:
 - Gerald Bauer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2020-12-06 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: webclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: csvreader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -44,7 +72,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.22'
-description: webget gem - yet (another) network client for world wide web (www) requests
+description: webget gem - a web (go get) crawler incl. web cache
 email: ruby-talk@ruby-lang.org
 executables: []
 extensions: []
@@ -60,13 +88,12 @@ files:
 - lib/webget.rb
 - lib/webget/version.rb
 - lib/webget/webcache.rb
-- lib/webget/webclient.rb
 - lib/webget/webget.rb
-homepage: https://github.com/rubycoco/fetcher
+homepage: https://github.com/rubycoco/webclient
 licenses:
 - Public Domain
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -83,9 +110,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
-summary: webget gem - yet (another) network client for world wide web (www) requests
+summary: webget gem - a web (go get) crawler incl. web cache
 test_files: []

data/lib/webget/webclient.rb

@@ -1,85 +0,0 @@
-
-class Webclient
-
-  class Response   # nested class - wrap Net::HTTP::Response
-    def initialize( response )
-      @response = response
-    end
-    def raw() @response; end
-
-
-    def text
-      # note: Net::HTTP will NOT set encoding UTF-8 etc.
-      # will be set to ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-      # thus, set/force encoding to utf-8
-      text = @response.body.to_s
-      text = text.force_encoding( Encoding::UTF_8 )
-      text
-    end
-
-    ## convenience helper; returns parsed json data
-    def json() JSON.parse( text ); end
-
-
-
-    class Headers # nested (nested) class
-      def initialize( response )
-        @response = response
-      end
-      def each( &blk )
-        @response.each_header do |key, value|  # Iterate all response headers
-          blk.call( key, value )
-        end
-      end
-    end
-    def headers() @headers ||= Headers.new( @response ); end
-
-    class Status  # nested (nested) class
-      def initialize( response )
-        @response = response
-      end
-      def code() @response.code.to_i; end
-      def ok?()  code == 200; end
-      def nok?() code != 200; end
-      def message() @response.message; end
-    end
-    def status() @status ||= Status.new( @response ); end
-  end # (nested) class Response
-
-
-def self.get( url, headers: {} )
-
-  uri = URI.parse( url )
-  http = Net::HTTP.new( uri.host, uri.port )
-
-  if uri.instance_of? URI::HTTPS
-    http.use_ssl     = true
-    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-  end
-
-  request = Net::HTTP::Get.new( uri.request_uri )
-
-  ### add (custom) headers if any
-  ##  check/todo: is there are more idiomatic way for Net::HTTP ???
-  ##   use
-  ##     request = Net::HTTP::Get.new( uri.request_uri, headers )
-  ##    why? why not?
-  ##  instead of e.g.
-  ##   request['X-Auth-Token'] = 'xxxxxxx'
-  ##   request['User-Agent']   = 'ruby'
-  ##   request['Accept']       = '*/*'
-  if headers && headers.size > 0
-    headers.each do |key,value|
-      request[ key ] = value
-    end
-  end
-
-
-  response = http.request( request )
-
-  ## note: return "unified" wrapped response
-  Response.new( response )
-end  # method self.get
-
-end  # class Webclient
-