webget 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e55270df44760e784f72743f62bed6e23f7c4f4
-  data.tar.gz: f2e529e98e53b294813fc00f8a14ba7c4d6bc58c
+  metadata.gz: 199c22178984860a75e48ecc68db3be4f3e1f210
+  data.tar.gz: ca9268c0e0650acee9125f333d567c8c3d65e3e2
 SHA512:
-  metadata.gz: a074c8c43fae86befca8c16b687a7701ebc7ae9e7e4a6ad3cee527178bd01a447440f1cbfaad6ff60f708fad43a5d70be90de9f20e576f8532658d3a47d08a5d
-  data.tar.gz: 968ae57d7a26b40f31ae17df384eb72f69fc83f2192c398a9982ea0549800c137bccc7bca0e4af669ee92aef3af7bccd43b48a6ee7d337d845f726d05dc68c46
+  metadata.gz: 02e990893588bfef74a704a1de9b0c361e184759ba81f5579aa6724254dd8321bdb900ae4c50556c9b4f6f7cd02ea0f7c3e13ce16d7eb4da36f679d4285bc134
+  data.tar.gz: 0ddefec72ef2900b43dc4160ddb97b1a94640c08fefcb1d4c0f5656e93d46e106ac6768ed030bbe905cbf03db0d4532ee89fcb87d956b894de2e2649e37db1cc

data/Rakefile

@@ -18,7 +18,8 @@ Hoe.spec 'webget' do
   self.history_file = 'CHANGELOG.md'
 
   self.extra_deps = [
-    ['webclient', '>= 0.1.0']
+    ['webclient', '>= 0.2.0'],
+    ['csvreader', '>= 1.2.4'],
   ]
 
   self.licenses = ['Public Domain']

data/lib/webget.rb

@@ -1,5 +1,9 @@
 require 'webclient'
 
+## more (our own) 3rd party libs
+require 'csvreader'
+
+
 ## our own code
 require 'webget/version'   # let version go first
 require 'webget/webcache'

data/lib/webget/version.rb

@@ -3,7 +3,7 @@ class Webget
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 2
-  PATCH = 1
+  PATCH = 2
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/webget/webcache.rb

@@ -54,16 +54,18 @@ module Webcache
  ### "interface" for "generic" cache storage (might be sqlite database or filesystem)
  def self.cache() @cache ||= DiskCache.new; end
 
- def self.record( url, response, format: 'html' )
-   cache.record( url, response, format: format );
+ def self.record( url, response, encoding: 'UTF-8', format: 'html' )
+   cache.record( url, response, encoding: encoding, format: format );
  end
  def self.cached?( url ) cache.cached?( url ); end
  class << self
    alias_method :exist?, :cached?
  end
- def self.url_to_id( url ) cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
+ def self.url_to_id( url )  cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
  def self.read( url )       cache.read( url );      end
  def self.read_json( url )  cache.read_json( url ); end
+ def self.read_csv( url )   cache.read_csv( url );  end
+
 
 
 class DiskCache
@@ -86,11 +88,17 @@ class DiskCache
     data
   end
 
+  def read_csv( url )
+    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    txt = File.open( body_path, 'r:utf-8' ) {|f| f.read }
+    data = CsvHash.parse( txt )
+    data
+  end
 
 
   ## add more save / put / etc. aliases - why? why not?
   ##  rename to record_html - why? why not?
-  def record( url, response, format: 'html' )
+  def record( url, response, encoding: 'UTF-8', format: 'html' )
 
     body_path = "#{Webcache.root}/#{url_to_path( url )}"
     meta_path = "#{body_path}.meta.txt"
@@ -102,14 +110,22 @@ class DiskCache
     puts "[cache] saving #{body_path}..."
 
     ## todo/check: verify content-type - why? why not?
+    ## note - for now respone.text always assume (converted) to utf8!!!!!!!!!
     if format == 'json'
       File.open( body_path, 'w:utf-8' ) {|f| f.write( JSON.pretty_generate( response.json )) }
+    elsif format == 'csv'
+      ## fix: newlines - always use "unix" style" - why? why not?
+      text = response.text( encoding: encoding ).gsub( "\r\n", "\n" )
+      File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
     else
-      ## note - for now always assume utf8!!!!!!!!!
-      File.open( body_path, 'w:utf-8' ) {|f| f.write( response.text ) }
+      text = response.text( encoding: encoding )
+      File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
     end
 
+
     File.open( meta_path, 'w:utf-8' ) do |f|
+      ## todo/check:
+      ##  do headers also need to converted (like text) if encoding is NOT utf-8 ???
       response.headers.each do |key, value|  # iterate all response headers
         f.write( "#{key}: #{value}" )
         f.write( "\n" )
@@ -164,6 +180,9 @@ class DiskCache
                          .gsub( '=', '~')
 
       req_path = "#{req_path}.html"
+    elsif host_dir.index( 'football-data.co.uk' )
+      req_path = req_path.sub( 'mmz4281/', '' )  # shorten - cut off mmz4281/
+      req_path = req_path.sub( 'new/', '' )      # shorten - cut off new/
     elsif host_dir.index( 'football-data.org' )
       req_path = req_path.sub( 'v2/', '' )  # shorten - cut off v2/
 

data/lib/webget/webget.rb

@@ -19,7 +19,7 @@ class Webget   # a web (go get) crawler
 
 
 
-  def self.call( url, headers: {} )  ## assumes json format
+  def self.call( url, headers: {} )  ## assumes json format (note - encoding always utf-8 by definition! - double check?)
     puts "  sleep #{config.sleep} sec(s)..."
     sleep( config.sleep )   ## slow down - sleep 3secs before each http request
 
@@ -40,8 +40,8 @@ class Webget   # a web (go get) crawler
     response
   end  # method self.call
 
-
-  def self.page( url, headers: {} )  ## assumes html format
+  ## todo/check: rename encoding to html/http-like charset - why? why not?
+  def self.page( url, encoding: 'UTF-8', headers: {} )  ## assumes html format
     puts "  sleep #{config.sleep} sec(s)..."
     sleep( config.sleep )   ## slow down - sleep 3secs before each http request
 
@@ -49,7 +49,8 @@ class Webget   # a web (go get) crawler
 
     if response.status.ok?  ## must be HTTP 200
       puts "#{response.status.code} #{response.status.message}"
-      Webcache.record( url, response )   ## assumes format: html (default)
+      Webcache.record( url, response,
+                       encoding: encoding  )   ## assumes format: html (default)
     else
       ## todo/check - log error
       puts "!! ERROR - #{response.status.code} #{response.status.message}:"
@@ -60,5 +61,30 @@ class Webget   # a web (go get) crawler
     response
   end  # method self.page
 
+
+  ## todo/check: rename to csv or file or records or - why? why not?
+  ## todo/check: rename encoding to html/http-like charset - why? why not?
+  def self.dataset( url, encoding: 'UTF-8', headers: {} )  ## assumes csv format
+    puts "  sleep #{config.sleep} sec(s)..."
+    sleep( config.sleep )   ## slow down - sleep 3secs before each http request
+
+    response = Webclient.get( url, headers: headers )
+
+    if response.status.ok?  ## must be HTTP 200
+      puts "#{response.status.code} #{response.status.message}"
+      Webcache.record( url, response,
+                       encoding: encoding,
+                       format:   'csv' )    ## pass along csv format - why? why not?
+    else
+      ## todo/check - log error
+      puts "!! ERROR - #{response.status.code} #{response.status.message}:"
+      pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
+    end
+
+    ## to be done / continued
+    response
+  end  # method self.dataset
+
+
 end  # class Webget
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webget
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-07 00:00:00.000000000 Z
+date: 2020-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webclient
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: csvreader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement