webget 0.2.0 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fbf6e29e3e80b0a0fe7b80ad8a4870c00601006c
-  data.tar.gz: c03efecb7b2362efef7392f26293c921f8422eca
+SHA256:
+  metadata.gz: 33af918725af96b367234f9280264a7cbc0a5c5fd965f4f5cd6da07a26ac43f4
+  data.tar.gz: a53145c4aa919e3073408decd06ebaa8262f70f2077a85bcb8c5cad1a295ec25
 SHA512:
-  metadata.gz: a3d026c4b3b6932eca4b22377ce1da9b08bd7836713c4e69c6a5fd28ba8de52449b28bca2be877e759eb73a30fb17b62966b779bed25a26fd072913abd284120
-  data.tar.gz: 5f7822300304672b181a3001fb5d5e4cb3ad2adc55853179895ba100e03b21a8ede90a800ad2cced34e3b13603162d22876acaf34abfabe61c665f1d821f0455
+  metadata.gz: 7459a96f8235fd8a9cec9d0f12512a3d8b26a4ef6b9f6f87b768ef75e5b806f32986ba4d507a78957f068b181abab4ec4386fd059f2f7a2c6c1f1098c96f82c4
+  data.tar.gz: 5272e8b6ce21110745d41b1c137aad4b92f99d7b1e519a1f0aade275e22d9bbc5f14b15c306eb9d92c81373519e3ef48e95b2e0dcf22367fd49514cceee5f265

data/Rakefile

@@ -18,7 +18,8 @@ Hoe.spec 'webget' do
   self.history_file = 'CHANGELOG.md'
 
   self.extra_deps = [
-    ['webclient', '>= 0.1.0']
+    ['webclient', '>= 0.2.0'],
+    ['csvreader', '>= 1.2.4'],
   ]
 
   self.licenses = ['Public Domain']

data/lib/webget.rb

@@ -1,5 +1,9 @@
 require 'webclient'
 
+## more (our own) 3rd party libs
+require 'csvreader'
+
+
 ## our own code
 require 'webget/version'   # let version go first
 require 'webget/webcache'

data/lib/webget/version.rb

@@ -3,7 +3,7 @@ class Webget
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 2
-  PATCH = 0
+  PATCH = 5
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/webget/webcache.rb

@@ -54,16 +54,24 @@ module Webcache
  ### "interface" for "generic" cache storage (might be sqlite database or filesystem)
  def self.cache() @cache ||= DiskCache.new; end
 
- def self.record( url, response, format: 'html' )
-   cache.record( url, response, format: format );
+ def self.record( url, response,
+                   path: nil,
+                   encoding: 'UTF-8',
+                   format: 'html' )
+   cache.record( url, response,
+                   path: path,
+                   encoding: encoding,
+                   format: format );
  end
  def self.cached?( url ) cache.cached?( url ); end
  class << self
    alias_method :exist?, :cached?
  end
- def self.url_to_id( url ) cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
+ def self.url_to_id( url )  cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
  def self.read( url )       cache.read( url );      end
  def self.read_json( url )  cache.read_json( url ); end
+ def self.read_csv( url )   cache.read_csv( url );  end
+
 
 
 class DiskCache
@@ -86,13 +94,22 @@ class DiskCache
     data
   end
 
+  def read_csv( url )
+    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    txt = File.open( body_path, 'r:utf-8' ) {|f| f.read }
+    data = CsvHash.parse( txt )
+    data
+  end
 
 
   ## add more save / put / etc. aliases - why? why not?
   ##  rename to record_html - why? why not?
-  def record( url, response, format: 'html' )
+  def record( url, response,
+              path: nil,
+              encoding: 'UTF-8',
+              format: 'html' )
 
-    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    body_path = "#{Webcache.root}/#{url_to_path( url, path: path )}"
     meta_path = "#{body_path}.meta.txt"
 
     ## make sure path exits
@@ -102,14 +119,23 @@ class DiskCache
     puts "[cache] saving #{body_path}..."
 
     ## todo/check: verify content-type - why? why not?
+    ## note - for now respone.text always assume (converted) to utf8!!!!!!!!!
     if format == 'json'
       File.open( body_path, 'w:utf-8' ) {|f| f.write( JSON.pretty_generate( response.json )) }
-    else
-      ## note - for now always assume utf8!!!!!!!!!
-      File.open( body_path, 'w:utf-8' ) {|f| f.write( response.text ) }
+    elsif format == 'csv'
+      ## fix: newlines - always use "unix" style" - why? why not?
+      ## fix:  use :newline => :universal option? translates to univeral "\n"
+      text = response.text( encoding: encoding ).gsub( "\r\n", "\n" )
+      File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
+    else   ## html or txt
+      text = response.text( encoding: encoding )
+      File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
     end
 
+
     File.open( meta_path, 'w:utf-8' ) do |f|
+      ## todo/check:
+      ##  do headers also need to converted (like text) if encoding is NOT utf-8 ???
       response.headers.each do |key, value|  # iterate all response headers
         f.write( "#{key}: #{value}" )
         f.write( "\n" )
@@ -125,7 +151,7 @@ class DiskCache
 
 
   ### helpers
-  def url_to_path( str )
+  def url_to_path( str, path: nil )
     ## map url to file path
     uri = URI.parse( str )
 
@@ -134,10 +160,14 @@ class DiskCache
     ##    always downcase for now (internet domain is case insensitive)
     host_dir = uri.host.downcase
 
-    ## "/this/is/everything?query=params"
-    ##   cut-off leading slash and
-    ##    convert query ? =
-    req_path = uri.request_uri[1..-1]
+    req_path = if path   ## use "custom" (file)path for cache storage if passed in
+                 path
+               else
+                ## "/this/is/everything?query=params"
+                ##   cut-off leading slash and
+                ##    convert query ? =
+                 uri.request_uri[1..-1]
+               end
 
 
 
@@ -151,6 +181,25 @@ class DiskCache
             puts "ERROR: expected request_uri for >#{host_dir}< ending with '/'; got: >#{req_path}<"
             exit 1
           end
+    elsif host_dir.index( 'tipp3.at' )
+      req_path = req_path.sub( '.jsp', '' )  # shorten - cut off .jsp extension
+
+      ##   change ? to -I-
+      ##   change = to ~
+      ##   Example:
+      ##   sportwetten/classicresults.jsp?oddsetProgramID=888
+      ##     =>
+      ##   sportwetten/classicresults-I-oddsetProgramID~888
+      req_path = req_path.gsub( '?', '-I-' )
+                         .gsub( '=', '~')
+
+      req_path = "#{req_path}.html"
+    elsif host_dir.index( 'fbref.com' )
+      req_path = req_path.sub( 'en/', '' )      # shorten - cut off en/
+      req_path = "#{req_path}.html"             # auto-add html extension
+    elsif host_dir.index( 'football-data.co.uk' )
+      req_path = req_path.sub( 'mmz4281/', '' )  # shorten - cut off mmz4281/
+      req_path = req_path.sub( 'new/', '' )      # shorten - cut off new/
     elsif host_dir.index( 'football-data.org' )
       req_path = req_path.sub( 'v2/', '' )  # shorten - cut off v2/
 
@@ -162,6 +211,11 @@ class DiskCache
                          .gsub( '/', '~~' )
                          .gsub( '=', '~')
 
+      req_path = "#{req_path}.json"
+    elsif host_dir.index( 'api.cryptokitties.co' )
+      ## for now always auto-add .json extensions e.g.
+      ##     kitties/1   => kitties/1.json
+      ##     cattributes => cattributes.json
       req_path = "#{req_path}.json"
     else
       ## no special rule

data/lib/webget/webget.rb

@@ -19,7 +19,7 @@ class Webget   # a web (go get) crawler
 
 
 
-  def self.call( url, headers: {} )  ## assumes json format
+  def self.call( url, headers: {} )  ## assumes json format (note - encoding always utf-8 by definition! - double check?)
     puts "  sleep #{config.sleep} sec(s)..."
     sleep( config.sleep )   ## slow down - sleep 3secs before each http request
 
@@ -40,8 +40,8 @@ class Webget   # a web (go get) crawler
     response
   end  # method self.call
 
-
-  def self.page( url, headers: {} )  ## assumes html format
+  ## todo/check: rename encoding to html/http-like charset - why? why not?
+  def self.page( url, encoding: 'UTF-8', headers: {} )  ## assumes html format
     puts "  sleep #{config.sleep} sec(s)..."
     sleep( config.sleep )   ## slow down - sleep 3secs before each http request
 
@@ -49,7 +49,8 @@ class Webget   # a web (go get) crawler
 
     if response.status.ok?  ## must be HTTP 200
       puts "#{response.status.code} #{response.status.message}"
-      Webcache.record( url, response )   ## assumes format: html (default)
+      Webcache.record( url, response,
+                       encoding: encoding  )   ## assumes format: html (default)
     else
       ## todo/check - log error
       puts "!! ERROR - #{response.status.code} #{response.status.message}:"
@@ -60,5 +61,54 @@ class Webget   # a web (go get) crawler
     response
   end  # method self.page
 
+
+  def self.text( url, path: nil, headers: {} )  ## assumes txt format
+    puts "  sleep #{config.sleep} sec(s)..."
+    sleep( config.sleep )   ## slow down - sleep 3secs before each http request
+
+    response = Webclient.get( url, headers: headers )
+
+    if response.status.ok?  ## must be HTTP 200
+      puts "#{response.status.code} #{response.status.message}"
+      ## note: like json assumes always utf-8 encoding for now !!!
+      Webcache.record( url, response,
+                       path: path,   ## optional "custom" (file)path for saving in cache
+                       format: 'txt' )
+    else
+      ## todo/check - log error
+      puts "!! ERROR - #{response.status.code} #{response.status.message}:"
+      pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
+    end
+
+    ## to be done / continued
+    response
+  end  # method self.text
+
+
+
+  ## todo/check: rename to csv or file or records or - why? why not?
+  ## todo/check: rename encoding to html/http-like charset - why? why not?
+  def self.dataset( url, encoding: 'UTF-8', headers: {} )  ## assumes csv format
+    puts "  sleep #{config.sleep} sec(s)..."
+    sleep( config.sleep )   ## slow down - sleep 3secs before each http request
+
+    response = Webclient.get( url, headers: headers )
+
+    if response.status.ok?  ## must be HTTP 200
+      puts "#{response.status.code} #{response.status.message}"
+      Webcache.record( url, response,
+                       encoding: encoding,
+                       format:   'csv' )    ## pass along csv format - why? why not?
+    else
+      ## todo/check - log error
+      puts "!! ERROR - #{response.status.code} #{response.status.message}:"
+      pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
+    end
+
+    ## to be done / continued
+    response
+  end  # method self.dataset
+
+
 end  # class Webget
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webget
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.5
 platform: ruby
 authors:
 - Gerald Bauer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-06 00:00:00.000000000 Z
+date: 2021-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webclient
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: csvreader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -79,7 +93,7 @@ homepage: https://github.com/rubycoco/webclient
 licenses:
 - Public Domain
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -96,9 +110,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: webget gem - a web (go get) crawler incl. web cache
 test_files: []