webget 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 02e86218c6a57c74af512d34f813f15ad11f71a9
-  data.tar.gz: 7342247ad14c9b7567b129a029ab01ed5147aac5
+SHA256:
+  metadata.gz: e171b577175334da7546a23003e44796c19da96b37c35a21947e5f025772cf19
+  data.tar.gz: c3a9a39e443edef2b702a15096cf5f5135a4aaccfc9fe1ef9fe8b0e9e9ce9296
 SHA512:
-  metadata.gz: 05a823f3a5918013c085099482055d42d29ed163a48a67563bda48da164d3e86a11969a1ab8c873895c9880f41321be3e05b904272c5d78a787ddb04924fb388
-  data.tar.gz: 7b412faa9fd940d788072c2dcac2eff90f68538c730b273f087f1195fd146673023eee3c32ae2c0e48fd28cfa575bc00dcc2a34795e7570272fb38c57a9dd7b7
+  metadata.gz: bf0c524fd19e2444a98df172ccf217d1c9a6201849dc3d947e2885229294271fc865df00cb329c2840a149a72e819facf224c07b2b555ef161874af2cb03c3e1
+  data.tar.gz: 44532a070b15e02cba5a861de875d25e16484d05dc1c0b8793b4e0874deaa94a464dafe8be047bc36284f9f1c9358760c6738f41e9828e2a15e0845e8f09c9d2

data/lib/webget/version.rb

@@ -3,7 +3,7 @@ class Webget
 
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 2
-  PATCH = 3
+  PATCH = 4
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/webget/webcache.rb

@@ -54,8 +54,14 @@ module Webcache
  ### "interface" for "generic" cache storage (might be sqlite database or filesystem)
  def self.cache() @cache ||= DiskCache.new; end
 
- def self.record( url, response, encoding: 'UTF-8', format: 'html' )
-   cache.record( url, response, encoding: encoding, format: format );
+ def self.record( url, response,
+                   path: nil,
+                   encoding: 'UTF-8',
+                   format: 'html' )
+   cache.record( url, response,
+                   path: path,
+                   encoding: encoding,
+                   format: format );
  end
  def self.cached?( url ) cache.cached?( url ); end
  class << self
@@ -98,9 +104,12 @@ class DiskCache
 
   ## add more save / put / etc. aliases - why? why not?
   ##  rename to record_html - why? why not?
-  def record( url, response, encoding: 'UTF-8', format: 'html' )
+  def record( url, response,
+              path: nil,
+              encoding: 'UTF-8',
+              format: 'html' )
 
-    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    body_path = "#{Webcache.root}/#{url_to_path( url, path: path )}"
     meta_path = "#{body_path}.meta.txt"
 
     ## make sure path exits
@@ -115,9 +124,10 @@ class DiskCache
       File.open( body_path, 'w:utf-8' ) {|f| f.write( JSON.pretty_generate( response.json )) }
     elsif format == 'csv'
       ## fix: newlines - always use "unix" style" - why? why not?
+      ## fix:  use :newline => :universal option? translates to univeral "\n"
       text = response.text( encoding: encoding ).gsub( "\r\n", "\n" )
       File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
-    else
+    else   ## html or txt
       text = response.text( encoding: encoding )
       File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
     end
@@ -141,7 +151,7 @@ class DiskCache
 
 
   ### helpers
-  def url_to_path( str )
+  def url_to_path( str, path: nil )
     ## map url to file path
     uri = URI.parse( str )
 
@@ -150,10 +160,14 @@ class DiskCache
     ##    always downcase for now (internet domain is case insensitive)
     host_dir = uri.host.downcase
 
-    ## "/this/is/everything?query=params"
-    ##   cut-off leading slash and
-    ##    convert query ? =
-    req_path = uri.request_uri[1..-1]
+    req_path = if path   ## use "custom" (file)path for cache storage if passed in
+                 path
+               else
+                ## "/this/is/everything?query=params"
+                ##   cut-off leading slash and
+                ##    convert query ? =
+                 uri.request_uri[1..-1]
+               end
 
 
 

data/lib/webget/webget.rb

@@ -62,6 +62,30 @@ class Webget   # a web (go get) crawler
   end  # method self.page
 
 
+  def self.text( url, path: nil, headers: {} )  ## assumes txt format
+    puts "  sleep #{config.sleep} sec(s)..."
+    sleep( config.sleep )   ## slow down - sleep 3secs before each http request
+
+    response = Webclient.get( url, headers: headers )
+
+    if response.status.ok?  ## must be HTTP 200
+      puts "#{response.status.code} #{response.status.message}"
+      ## note: like json assumes always utf-8 encoding for now !!!
+      Webcache.record( url, response,
+                       path: path,   ## optional "custom" (file)path for saving in cache
+                       format: 'txt' )
+    else
+      ## todo/check - log error
+      puts "!! ERROR - #{response.status.code} #{response.status.message}:"
+      pp response.raw  ## note: dump inner (raw) response (NOT the wrapped)
+    end
+
+    ## to be done / continued
+    response
+  end  # method self.text
+
+
+
   ## todo/check: rename to csv or file or records or - why? why not?
   ## todo/check: rename encoding to html/http-like charset - why? why not?
   def self.dataset( url, encoding: 'UTF-8', headers: {} )  ## assumes csv format

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webget
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Gerald Bauer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-20 00:00:00.000000000 Z
+date: 2020-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webclient
@@ -93,7 +93,7 @@ homepage: https://github.com/rubycoco/webclient
 licenses:
 - Public Domain
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -110,9 +110,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: webget gem - a web (go get) crawler incl. web cache
 test_files: []