webget 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de0060acabe176d1a11a4f2e36e8bee5090083c4907b75d9b22e9cbb9c7b22e2
-  data.tar.gz: fb64c2b1294932b00fa401af006cd27b263b255789bad75e4ef62bec5598fd74
+  metadata.gz: bb98b8d5740ff961389adbfcda8b1a461c1aa0f8fe6f9cc748ac8411c8559610
+  data.tar.gz: ccb159f52cc457f6a83e2fceabdab1d197018a58347e765b32726651a5985f01
 SHA512:
-  metadata.gz: d1f1653c68729e7d609c3c848e0146de9668cd2932ddab2e794ab361d3f786603a10c7586bcf0029de4955255803987000b2fb35b0d1e14cfa35d0582f919be7
-  data.tar.gz: c6cff08b2f683bb5b8e39607735250f954115244be3e19dfcdeb29fd2d0a0f0a27be1fed5a2add9a1b75a9cbe81dbafea1e80f06b0a1bdd93d4bd3bd12d5a9a1
+  metadata.gz: 1d469ba45efeaf35d95d241e0521e2b447779ac88e8d5788e5e9599a759c9d7ad0e25c9ecff0a263d2755c9d1cf757196f0511a7172d17788652ab37c1a25f45
+  data.tar.gz: 87122ab93ceabb553b289f3f2ebc9343ee8d063b38ddd7f142e0534543acd9bb152aa8903b94b442b5d9249a05ffaaefcdd34b2ad41701540d0970d428551d0d

data/CHANGELOG.md

@@ -1,6 +1,4 @@
-### 0.3.1
-
+### 0.3.3
 ### 0.0.1 / 2020-10-04
 
 * Everything is new. First release.
-

data/Manifest.txt

@@ -5,4 +5,7 @@ Rakefile
 lib/webget.rb
 lib/webget/version.rb
 lib/webget/webcache.rb
+lib/webget/webcache_disk-rewrite.rb
+lib/webget/webcache_disk.rb
+lib/webget/webcache_headers.rb
 lib/webget/webget.rb

data/README.md

@@ -2,8 +2,8 @@
 
 webget gem - a web (go get) crawler incl. web cache
 
-* home  :: [github.com/rubycoco/webclient](https://github.com/rubycoco/webclient)
-* bugs  :: [github.com/rubycoco/webclient/issues](https://github.com/rubycoco/webclient/issues)
+* home  :: [github.com/rubycocos/webclient](https://github.com/rubycocos/webclient)
+* bugs  :: [github.com/rubycocos/webclient/issues](https://github.com/rubycocos/webclient/issues)
 * gem   :: [rubygems.org/gems/webget](https://rubygems.org/gems/webget)
 * rdoc  :: [rubydoc.info/gems/webget](http://rubydoc.info/gems/webget)
 
@@ -16,4 +16,4 @@ TBD
 ## License
 
 The `webget` scripts are dedicated to the public domain.
-Use it as you please with no restrictions whatsoever.
+Use  as you please with no restrictions whatsoever.

data/Rakefile

@@ -8,7 +8,7 @@ Hoe.spec 'webget' do
   self.summary = 'webget gem - a web (go get) crawler incl. web cache'
   self.description = summary
 
-  self.urls    = { home: 'https://github.com/rubycoco/webclient' }
+  self.urls    = { home: 'https://github.com/rubycocos/webclient' }
 
   self.author  = 'Gerald Bauer'
   self.email   = 'gerald.bauer@gmail.com'
@@ -18,15 +18,14 @@ Hoe.spec 'webget' do
   self.history_file = 'CHANGELOG.md'
 
   self.extra_deps = [
-    ['webclient', '>= 0.2.0'],
-    ['csvreader', '>= 1.2.4'],
+    ['webclient', '>= 0.3.1'],
     ['cocos'],
   ]
 
   self.licenses = ['Public Domain']
 
   self.spec_extras = {
-   required_ruby_version: '>= 2.2.2'
+   required_ruby_version: '>= 3.1.0'
   }
 
 end

data/lib/webget/version.rb

@@ -2,7 +2,7 @@
 class Webget
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 3
-  PATCH = 1
+  PATCH = 3
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version
@@ -18,4 +18,3 @@ class Webget
     File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
   end
 end  # module Webget
-

data/lib/webget/webcache.rb

@@ -2,6 +2,7 @@
 
 module Webcache
 
+
   #####
   # copied from props gem, see Env.home
   #    - https://github.com/rubycoco/props/blob/master/props/lib/props/env.rb
@@ -26,7 +27,6 @@ module Webcache
     ## note: use File.expand_path to "unify" path e.g
     ##  C:\Users\roman  becomes
     ##  C:/Users/roman
-
     File.expand_path( path )
  end
 
@@ -34,7 +34,14 @@ module Webcache
  class Configuration
     ## root directory - todo/check: find/use a better name - why? why not?
     def root()       @root || "#{Webcache.home}/.cache"; end
-    def root=(value) @root = value; end
+    def root=(path)
+       ## note: use File.expand_path to "unify" and auto-expand path
+       ##                to make sure always absolute
+       ## e.g
+       ##  C:\Users\roman  becomes
+       ##  C:/Users/roman
+        @root = File.expand_path(path)
+    end
  end # class Configuration
 
 
@@ -49,186 +56,51 @@ module Webcache
  ## add "high level" root convenience helpers
  ##   use delegate helper - why? why not?
  def self.root()       config.root; end
- def self.root=(value) config.root = value; end
+ def self.root=(path)  config.root = path; end
 
 
- ### "interface" for "generic" cache storage (might be sqlite database or filesystem)
- def self.cache() @cache ||= DiskCache.new; end
-
- def self.record( url, response,
-                   path: nil,
-                   encoding: 'UTF-8',
-                   format: 'html' )
-   cache.record( url, response,
-                   path: path,
-                   encoding: encoding,
-                   format: format );
- end
- def self.cached?( url ) cache.cached?( url ); end
- class << self
-   alias_method :exist?, :cached?
- end
- def self.url_to_id( url )  cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
- def self.read( url )       cache.read( url );      end
- def self.read_json( url )  cache.read_json( url ); end
- def self.read_csv( url )   cache.read_csv( url );  end
 
 
 
-class DiskCache
-  def cached?( url )
-    body_path = "#{Webcache.root}/#{url_to_path( url )}"
-    File.exist?( body_path )
-  end
-  alias_method :exist?, :cached?
+ ### "interface" for "generic" cache storage (might be sqlite database or filesystem)
+ def self.cache() @cache ||= DiskCache.new; end
 
+ def self.record( url, response, format: )  ## html|txt|csv|json|etc.
+  ##  note - (text) encoding_user MUST get passed along in response obj/wrapper
+  ##            response._encoding_user = encoding! !!
+   cache.record( url, response, format: format );
+ end
 
-  def read( url )
-    body_path = "#{Webcache.root}/#{url_to_path( url )}"
-    File.open( body_path, 'r:utf-8' ) {|f| f.read }
-  end
+ def self.cached?( url ) cache.cached?( url ); end
 
-  def read_json( url )
-    body_path = "#{Webcache.root}/#{url_to_path( url )}"
-    txt = File.open( body_path, 'r:utf-8' ) {|f| f.read }
-    data = JSON.parse( txt )
-    data
-  end
+ ##  def self.url_to_id( url )  cache.url_to_id( url ); end  ## todo/check: rename to just id or something - why? why not?
 
-  def read_csv( url )
-    body_path = "#{Webcache.root}/#{url_to_path( url )}"
-    txt = File.open( body_path, 'r:utf-8' ) {|f| f.read }
-    data = CsvHash.parse( txt )
-    data
-  end
 
+ def self.read( url )       cache.read( url );      end
+ def self.read_json( url )  cache.read_json( url ); end
+ def self.read_csv( url )   cache.read_csv( url );  end
 
-  ## add more save / put / etc. aliases - why? why not?
-  ##  rename to record_html - why? why not?
-  def record( url, response,
-              path: nil,
-              encoding: 'UTF-8',
-              format: 'html' )
-
-    body_path = "#{Webcache.root}/#{url_to_path( url, path: path )}"
-    meta_path = "#{body_path}.meta.txt"
-
-    ## make sure path exits
-    FileUtils.mkdir_p( File.dirname( body_path ) )
-
-
-    puts "[cache] saving #{body_path}..."
-
-    ## todo/check: verify content-type - why? why not?
-    ## note - for now respone.text always assume (converted) to utf8!!!!!!!!!
-    ##
-    ## fix: newlines - always use "unix" style" - why? why not?
-    ## fix:  use :newline => :universal option? translates to univeral "\n"
-    if format == 'json'
-      File.open( body_path, 'w:utf-8' ) {|f| f.write( JSON.pretty_generate( response.json )) }
-    elsif format == 'csv'
-      ## fix: newlines - always use "unix" style" - why? why not?
-      ## fix:  use :newline => :universal option? translates to univeral "\n"
-      text = response.text( encoding: encoding ).gsub( "\r\n", "\n" )
-      File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
-    else   ## html or txt
-      text = response.text( encoding: encoding ).gsub( "\r\n", "\n" )
-      File.open( body_path, 'w:utf-8' ) {|f| f.write( text ) }
-    end
 
+#### new - read (cached) meta data
+##    todo/check - find a better/different name - why? why not?
+##       e.g. read_headers or simply meta or headers or such
+  def self.read_meta( url ) cache.read_meta( url ); end
 
-    File.open( meta_path, 'w:utf-8' ) do |f|
-      ## todo/check:
-      ##  do headers also need to converted (like text) if encoding is NOT utf-8 ???
-      response.headers.each do |key, value|  # iterate all response headers
-        f.write( "#{key}: #{value}" )
-        f.write( "\n" )
-      end
-    end
+  ## add convenience expire (shortcut) helpers
+  def self.expired?( url, expires_in: Time.now.utc-60*60*12 )
+     if cached?( url )
+       meta = read_meta( url )
+       meta.expired?( expires_in )
+     else
+       true  # note - not in cache; expired by default
+     end
   end
+  def self.expired_in_12h?( url ) expired?( url, expires_in: Time.now.utc-60*60*12 ); end
+  def self.expired_in_24h?( url ) expired?( url, expires_in: Time.now.utc-60*60*24 ); end
 
 
-
-  ### note: use file path as id for DiskCache  (is different for DbCache/SqlCache?)
-  ##    use file:// instead of disk:// - why? why not?
-  def url_to_id( str ) "disk://#{url_to_path( str )}"; end
-
-
-  ### helpers
-  def url_to_path( str, path: nil )
-    ## map url to file path
-    uri = URI.parse( str )
-
-    ## note: ignore scheme (e.g. http/https)
-    ##         and  post  (e.g. 80, 8080, etc.) for now
-    ##    always downcase for now (internet domain is case insensitive)
-    host_dir = uri.host.downcase
-
-    req_path = if path   ## use "custom" (file)path for cache storage if passed in
-                 path
-               else
-                ## "/this/is/everything?query=params"
-                ##   cut-off leading slash and
-                ##    convert query ? =
-                 uri.request_uri[1..-1]
-               end
-
-
-
-    ### special "prettify" rule for weltfussball
-    ##   /eng-league-one-2019-2020/  => /eng-league-one-2019-2020.html
-    if host_dir.index( 'weltfussball.de' ) ||
-       host_dir.index( 'worldfootball.net' )
-          if req_path.end_with?( '/' )
-             req_path = "#{req_path[0..-2]}.html"
-          else
-            puts "ERROR: expected request_uri for >#{host_dir}< ending with '/'; got: >#{req_path}<"
-            exit 1
-          end
-    elsif host_dir.index( 'tipp3.at' )
-      req_path = req_path.sub( '.jsp', '' )  # shorten - cut off .jsp extension
-
-      ##   change ? to -I-
-      ##   change = to ~
-      ##   Example:
-      ##   sportwetten/classicresults.jsp?oddsetProgramID=888
-      ##     =>
-      ##   sportwetten/classicresults-I-oddsetProgramID~888
-      req_path = req_path.gsub( '?', '-I-' )
-                         .gsub( '=', '~')
-
-      req_path = "#{req_path}.html"
-    elsif host_dir.index( 'fbref.com' )
-      req_path = req_path.sub( 'en/', '' )      # shorten - cut off en/
-      req_path = "#{req_path}.html"             # auto-add html extension
-    elsif host_dir.index( 'football-data.co.uk' )
-      req_path = req_path.sub( 'mmz4281/', '' )  # shorten - cut off mmz4281/
-      req_path = req_path.sub( 'new/', '' )      # shorten - cut off new/
-    elsif host_dir.index( 'football-data.org' )
-      ##  req_path = req_path.sub( 'v2/', '' )  # shorten - cut off v2/
-
-      ## flattern - make a file path - for auto-save
-      ##   change ? to -I-
-      ##   change / to ~~
-      ##   change = to ~
-      req_path = req_path.gsub( '?', '-I-' )
-                         .gsub( '/', '~~' )
-                         .gsub( '=', '~')
-
-      req_path = "#{req_path}.json"
-    elsif host_dir.index( 'api.cryptokitties.co' )
-      ## for now always auto-add .json extensions e.g.
-      ##     kitties/1   => kitties/1.json
-      ##     cattributes => cattributes.json
-      req_path = "#{req_path}.json"
-    else
-      ## no special rule
-    end
-
-    page_path = "#{host_dir}/#{req_path}"
-    page_path
+  class << self
+    alias_method :exist?,         :cached?
+    alias_method :expired_in_1d?, :expired_in_24h?
   end
-end # class DiskCache
-
-
 end  # module Webcache

data/lib/webget/webcache_disk-rewrite.rb

@@ -0,0 +1,86 @@
+
+module Webcache
+class DiskCache
+
+####
+##  todo/fix - make rewrite_path configurable "pipeline"
+##          lets you auto-add more rewriters
+##         and keep this code "generic"
+##    move rewriters "downstream" into "userland" ??
+
+def rewrite_path( host, req_path )
+
+        ### special "prettify" rule for weltfussball
+    ##   /eng-league-one-2019-2020/  => /eng-league-one-2019-2020.html
+
+    ### todo/fix - move rules downstream to user - why? why not?
+
+    if host.include?( 'uefa.com' ) ||
+       host.include?( 'kicker.de' ) ||
+       host.include?( 'kicker.at' )
+      if req_path.end_with?( '/' )
+        req_path = "#{req_path[0..-2]}.html"
+      else
+        puts "ERROR: expected request_uri for >#{host}< ending with '/'; got: >#{req_path}<"
+        exit 1
+      end
+    elsif host.include?( 'weltfussball.de' ) ||
+          host.include?( 'worldfootball.net' )
+          if req_path.end_with?( '/' )
+             req_path = "#{req_path[0..-2]}.html"
+          else
+            puts "ERROR: expected request_uri for >#{host}< ending with '/'; got: >#{req_path}<"
+            exit 1
+          end
+    elsif host.include?( 'tipp3.at' )
+      req_path = req_path.sub( '.jsp', '' )  # shorten - cut off .jsp extension
+
+      ##   change ? to -I-
+      ##   change = to ~
+      ##   Example:
+      ##   sportwetten/classicresults.jsp?oddsetProgramID=888
+      ##     =>
+      ##   sportwetten/classicresults-I-oddsetProgramID~888
+      req_path = req_path.gsub( '?', '-I-' )
+                         .gsub( '=', '~')
+
+      req_path = "#{req_path}.html"
+    elsif host.include?( 'fbref.com' )
+      req_path = req_path.sub( 'en/', '' )      # shorten - cut off en/
+      req_path = "#{req_path}.html"             # auto-add html extension
+    elsif host.include?( 'football-data.co.uk' )
+      req_path = req_path.sub( 'mmz4281/', '' )  # shorten - cut off mmz4281/
+      req_path = req_path.sub( 'new/', '' )      # shorten - cut off new/
+    elsif host.include?( 'football-data.org' )
+      ##  req_path = req_path.sub( 'v2/', '' )  # shorten - cut off v2/
+
+      ## flattern - make a file path - for auto-save
+      ##   change ? to -I-
+      ##   change / to ~~
+      ##   change = to ~
+      req_path = req_path.gsub( '?', '-I-' )
+                         .gsub( '/', '~~' )
+                         .gsub( '=', '~')
+
+      req_path = "#{req_path}.json"
+    elsif host.include?( 'api-sports.io' )
+      req_path = req_path.gsub( '?', '-I-' )
+                         .gsub( '&', '~~' )   ### check if & present?
+                         .gsub( '=', '~')
+
+      req_path = "#{req_path}.json"
+    elsif host.include?( 'api.cryptokitties.co' )
+      ## for now always auto-add .json extensions e.g.
+      ##     kitties/1   => kitties/1.json
+      ##     cattributes => cattributes.json
+      req_path = "#{req_path}.json"
+    else
+      ## no special rule
+    end
+
+    req_path
+end  # method rewrite_path
+
+
+end  # class DiskCache
+end # module Webcache

data/lib/webget/webcache_disk.rb

@@ -0,0 +1,244 @@
+
+module Webcache
+
+
+class DiskCache     ### todo/check - change to Disk - why? why not?
+
+
+  def _body_path( url ) "#{Webcache.root}/#{url_to_path( url )}"; end
+  def _meta_path( url ) "#{Webcache.root}/#{url_to_path( url )}.meta.txt"; end
+
+
+  def cached?( url )
+    body_path = _body_path( url )
+    exist =  File.exist?( body_path )
+
+=begin
+##  not really working - check back later
+###   The catch on Windows
+##   On Windows, File.realpath does NOT normalize casing to the on-disk canonical case.
+##
+## note - on windows - file.exist? is case-insensitive
+##         use the strict: true flag if you want to enforce case-sensitive exists checks on windows!!!
+##   On Windows, File.realpath returns the "true" path
+##    as stored on the disk with the correct casing.
+##    If the path you provide doesn't match the casing of the real path,
+##     you know the match was case-insensitive.
+    if exist && strict
+     exist =  File.realpath(body_path) == File.expand_path(body_path)
+    end
+=end
+
+    exist
+  end
+  alias_method :exist?, :cached?
+
+
+
+  ### fix-fix-fix
+  ##    change to read_txt/read_text/read_html
+  ##  plus add
+  ##     read_blob/read_bin(ary) !!!
+  def read( url )
+    _read_utf8(_body_path( url ))
+  end
+
+
+
+  def read_json( url )
+    txt = _read_utf8(_body_path( url ))
+    data = JSON.parse( txt )
+    data
+  end
+
+  def read_csv( url )
+    txt = _read_utf8(_body_path( url ))
+    data = CsvHash.parse( txt )
+    data
+  end
+
+
+  def read_meta( url )
+    txt = _read_utf8(_meta_path( url ))
+    data = Headers.parse( txt )
+    data
+  end
+
+
+
+
+  ## add more save / put / etc. aliases - why? why not?
+  ##  rename to record_html - why? why not?
+
+  def record( url, response, format: )
+
+    ###
+    ## note - encoding_user MUST be passed along with response (wrapper) obj
+    ##           e.g.  response._encoding_user = encoding ??
+    ##                   see Webget.page|text|dataset|etc.
+
+
+    ## todo/check - use rel_path or local_path or such??
+    save_path = url_to_path( url )
+
+    body_path = _body_path( url )
+    meta_path = _meta_path( url )   ## is _body_path + ".meta.txt"
+
+    ## make sure path exits
+    FileUtils.mkdir_p( File.dirname( body_path ) )
+
+
+    puts "[cache] saving #{body_path}..."
+
+    ## todo/check: verify content-type - why? why not?
+    ## note - for now respone.text always assume (converted) to utf8!!!!!!!!!
+
+    if format == 'json'
+      _write_utf8( body_path, JSON.pretty_generate( response.json ))
+      x_encoding        = nil   ## for now do not track; always assume  UTF-8
+      x_encoding_source = nil
+      x_encoding_valid  = nil
+      x_ascii_only      = nil
+      x_8bit            = nil
+      x_utf8_replace    = nil
+    else   ## html,  txt or csv
+      _write_utf8( body_path, response.text )
+
+      x_encoding        = response._text_encoding
+      x_encoding_source = response._text_encoding_source
+      x_encoding_valid  = response._text_encoding_valid  # true|false or nil (undef)
+      x_ascii_only      = response._text_ascii_only
+      x_8bit            = response._text_8bit
+      x_utf8_replace    = response._text_utf8_replace
+    end
+
+
+    ### fix-fix-fix  -- add support for binary/image formats
+    ##     e.g. bin|gif|jpg|etc  - why? why not?
+
+    ####
+    ## get file size in bytes
+    ##     or use File.stat( body_path ).size (using File::Stat) ??
+    x_size    = File.size( body_path )
+
+
+
+      ## todo/check:
+      ##  do headers also need to converted (like text) if encoding is NOT utf-8 ???
+
+
+      #### add our own custom headers first!!
+      ##     change x-save to x-filename or ??
+      ##     change to x-7bit-only or x-ascii7bit or x-ascii7bit-only or ??
+      ##     change x-size to x-bytesize or ??
+      ##     change x-8bit  to ???
+
+      ### start w/ comment line
+      ###   uncomment - http status - why? why not?
+      buf = String.new
+      buf << "# fetched on #{Time.now.utc}\n"
+      buf << "# HTTP/#{response.version} #{response.status.code} #{response.status.message}\n"
+      buf << "\n"
+
+      buf << "x-url: #{url}\n"
+      buf << "x-encoding: #{x_encoding}\n"                 if x_encoding
+      buf << "x-encoding-source: #{x_encoding_source}\n"   if x_encoding_source
+      buf << "x-encoding-valid: #{x_encoding_valid}\n"     if x_encoding_valid
+      buf << "x-ascii-only: #{x_ascii_only}\n"             if x_ascii_only
+      buf << "x-8bit: #{x_8bit}\n"                         if x_8bit
+      buf << "x-utf8-replace: #{x_utf8_replace}\n"         if x_utf8_replace
+      buf << "x-save: #{save_path}\n"
+      buf << "x-size: #{x_size}\n"
+      buf << "x-format: #{format}\n"      ## e.g. json|html|csv|etc.
+      buf << "\n"
+
+      # iterate all response headers
+      response.headers.each do |key, value|
+        buf << "#{key}: #{value}\n"
+      end
+
+      _write_utf8( meta_path, buf )
+  end
+
+
+
+  ### note: use file path as id for DiskCache  (is different for DbCache/SqlCache?)
+  ##    use file:// instead of disk:// - why? why not?
+  ##    def url_to_id( str ) "disk://#{url_to_path( str )}"; end
+
+
+  ### helpers
+
+  def url_to_path( str )
+    ## map url to file path
+    uri = URI( str )       ## URI() same as URI.parse()
+
+    ## note: ignore scheme (e.g. http/https)
+    ##         and  post  (e.g. 80, 8080, etc.) for now
+    ##    always downcase for now (internet domain is case insensitive)
+    host_dir = uri.host.downcase
+
+    ## "/this/is/everything?query=params"
+    ##   cut-off leading slash and
+    ##    convert query ? =
+    ##   check if [1..] is sames as [1..-1]
+    req_path =   rewrite_path( host_dir, uri.request_uri[1..] )
+
+
+    page_path = "#{host_dir}/#{req_path}"
+    page_path
+  end
+
+
+
+  def _read_utf8( path )
+##  note - by default ruby on windows (automagically)
+##          translates \r\n newlines to \n (universal/unix-style)
+##                  when read files!!!!
+##   note - only handles \r\n  (not "legacy" mac classic-style \r)
+
+    File.open( path, 'r:utf-8' ) {|f| f.read }
+  end
+
+  def _write_utf8( path, text )
+##  write out utf8 (always use "universal" newlines on any platform)
+##    todo / fix -   add  universial or such to open too ?
+##
+## fix: newlines - always use "unix" style" - why? why not?
+## fix:  use :newline => :universal option? translates to univeral "\n"
+##
+##
+##  The universal_newline: true flag forces Ruby to look through the string and
+##   convert both Windows-style (\r\n) and old Mac-style (\r) newlines
+##  into the standard Unix newline (\n)
+##
+##  File.open("output.txt", "w", universal_newline: true) do |file|
+##     file.write(content)
+##  end
+##
+##  mixed_string = "Line one\r\nLine two\rLine three\n"
+##
+##   Converts all \r\n and \r into \n
+##     clean_string = mixed_string.encode(universal_newline: true)
+##
+##  Why Use encode Instead of gsub?
+##  While many developers use regular expressions like .gsub(/\r\n?/, "\n"),
+##   using .encode is highly preferred because:
+##  Edge-case Safety: It is an internal, optimized C-level implementation
+##   that handles mixed and broken newline edges perfectly.
+##  Encoding Preservation: It seamlessly preserves the existing character encoding
+##    (e.g., UTF-8) of your string.
+##
+##  was -  text  = text.gsub( "\r\n", "\n" )
+##
+##  note - by default ruby on windows (automagically) translates newlines to \r\n (crlf)!!!
+##          thus, always use/ add universal_newline flag!!!
+
+    File.open( path, 'w:utf-8', universal_newline: true ) do |f|
+        f.write( text )
+    end
+  end
+end # class DiskCache
+
+
+end  ## module Webcache

data/lib/webget/webcache_headers.rb

@@ -0,0 +1,79 @@
+
+module Webcache
+
+  ##############################
+  # nested class for convenience access to (meta) headers
+
+
+  class Headers      ## todo/check - rename to Meta or such - why? why not?
+
+
+    ##  def self.read( path ) parse(read_text( path )); end
+
+
+
+    def self.parse( txt )
+      data = {}
+      txt.each_line do |line|
+         line = line.strip
+         next  if line.empty? || line.start_with?( '#' )
+
+         key, value = line.split( ':', 2 )  ## split on first colon
+
+         ##  todo/fix: deal with possible duplicate header keys!!
+         ##   if duplicate do NOT replease, add with leading ", " comma-separated!!!
+         ##
+         ##  check if multi-line headers are possible!!!
+         ##   and than may turn value into array of values - why? why not?
+         ##    or auto-add
+
+
+         ## note - always downcase keys for now
+         ##  and strip value from leading and trailing spaces
+         data[ key.strip.downcase ] = value.strip
+      end
+      new( data )
+    end
+
+
+
+    def initialize( data )
+      @data = data
+    end
+
+    def to_h() @data; end
+    def [](key) @data[key]; end
+
+    def each( &blk )
+      @data.each do |key, value|
+        blk.call( key, value )
+      end
+    end
+
+
+
+    def date
+       ## return date header
+       ##  parses the time as RFC 1123 date of HTTP-date defined by RFC 2616:
+       ##    day-of-week, DD month-name CCYY hh:mm:ss GMT
+       ##   !!! Note that the result is always UTC (GMT). !!!
+       ##   e.g. Sun, 19 May 2024 15:15:34 GMT
+       ##        Mon, 10 Jun 2024 15:58:16 GMT
+       @date ||= Time.httpdate( @data['date'] )
+       @date
+    end
+
+    ## default to 12h (60secs*60min*12h)
+    def expired?( expires_in_date=Time.now.utc-60*60*12 )
+      ## pp expires_in_date
+      expires_in_date > date
+    end
+
+    ## add convenience helpers - why? why not?
+    def expired_in_12h?() expired?( Time.now.utc-60*60*12 ); end
+    def expired_in_24h?() expired?( Time.now.utc-60*60*24 ); end
+    alias_method :expired_in_1d?, :expired_in_24h?
+  end # class Headers
+
+
+end  ## module Webcache

data/lib/webget/webget.rb

@@ -22,16 +22,15 @@ class Webget   # a web (go get) crawler
 
 
 
-  ## note - assumes json format 
+  ## note - assumes json format
   ##   encoding always utf-8 by definition! - double check?)
-  def self.call( url, headers: {} )  
+  def self.call( url, headers: {} )
     response = _get( url, headers: headers )
 
     if response.status.ok?  ## must be HTTP 200
       puts "#{response.status.code} #{response.status.message}"
       ## note: use format json for pretty printing and parse check!!!!
-      Webcache.record( url, response,
-                       format: 'json' )
+      Webcache.record( url, response, format: 'json' )
     else
       ## todo/check - log error
       puts "!! HTTP ERROR - #{response.status.code} #{response.status.message}:"
@@ -40,17 +39,22 @@ class Webget   # a web (go get) crawler
 
     ## to be done / continued
     response
-  end  # method self.call
+  end  # method self._get_and_record
+
+
+
 
   ## todo/check: rename encoding to html/http-like charset - why? why not?
   ##   check encoding UTF-8 or utf-8  - makes a difference?
-  def self.page( url, encoding: 'UTF-8', headers: {} )  ## assumes html format
+  ##  note - default text encoding "upstream" in webclient for text is UTF-8!!
+
+  def self.page( url, encoding: nil, headers: {} )  ## assumes html format
     response = _get( url, headers: headers )
+    response._encoding_user = encoding
 
     if response.status.ok?  ## must be HTTP 200
       puts "#{response.status.code} #{response.status.message}"
-      Webcache.record( url, response,
-                       encoding: encoding  )   ## assumes format: html (default)
+      Webcache.record( url, response, format: 'html'  )
     else
       ## todo/check - log error
       puts "!! HTTP ERROR - #{response.status.code} #{response.status.message}:"
@@ -63,15 +67,13 @@ class Webget   # a web (go get) crawler
 
 
   ## assumes txt format
-  def self.text( url, path: nil, headers: {} )  
+  def self.text( url, encoding: nil, headers: {} )
     response = _get( url, headers: headers )
+    response._encoding_user = encoding
 
     if response.status.ok?  ## must be HTTP 200
       puts "#{response.status.code} #{response.status.message}"
-      ## note: like json assumes always utf-8 encoding for now !!!
-      Webcache.record( url, response,
-                       path: path,   ## optional "custom" (file)path for saving in cache
-                       format: 'txt' )
+      Webcache.record( url, response, format: 'txt' )
     else
       ## todo/check - log error
       puts "!! HTTP ERROR - #{response.status.code} #{response.status.message}:"
@@ -86,14 +88,13 @@ class Webget   # a web (go get) crawler
 
   ## todo/check: rename to csv or file or records or - why? why not?
   ## todo/check: rename encoding to html/http-like charset - why? why not?
-  def self.dataset( url, encoding: 'UTF-8', headers: {} )  ## assumes csv format
+  def self.dataset( url, encoding: nil, headers: {} )  ## assumes csv format
     response = _get( url, headers: headers )
+    response._encoding_user = encoding
 
     if response.status.ok?  ## must be HTTP 200
       puts "#{response.status.code} #{response.status.message}"
-      Webcache.record( url, response,
-                       encoding: encoding,
-                       format:   'csv' )    ## pass along csv format - why? why not?
+      Webcache.record( url, response, format: 'csv' )    ## pass along csv format - why? why not?
     else
       ## todo/check - log error
       puts "!! HTTP ERROR - #{response.status.code} #{response.status.message}:"
@@ -106,8 +107,9 @@ class Webget   # a web (go get) crawler
 
 
 
+
   ####
-  ##  private helpers 
+  ##  private helpers
   ##   make private - why? why not?
   def self._get( url, headers: {} )
      @@requests ||= 0     ## track number of requests
@@ -121,4 +123,3 @@ class Webget   # a web (go get) crawler
      Webclient.get( url, headers: headers )  ## returns respone
   end
 end  # class Webget
-

data/lib/webget.rb

@@ -10,7 +10,13 @@ require 'cocos'   # - note - cococs incl. webclient & cvsreader  !!!!
 
 ## our own code
 require_relative 'webget/version'   # let version go first
+
+require_relative 'webget/webcache_headers'
+require_relative 'webget/webcache_disk'
+require_relative 'webget/webcache_disk-rewrite'
 require_relative 'webget/webcache'
+
+
 require_relative 'webget/webget'
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webget
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-04 00:00:00.000000000 Z
+date: 2026-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webclient
@@ -16,28 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
-- !ruby/object:Gem::Dependency
-  name: csvreader
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.2.4
+        version: 0.3.1
 - !ruby/object:Gem::Dependency
   name: cocos
   requirement: !ruby/object:Gem::Requirement
@@ -78,14 +64,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.1'
+        version: '4.2'
 description: webget gem - a web (go get) crawler incl. web cache
 email: gerald.bauer@gmail.com
 executables: []
@@ -102,8 +88,11 @@ files:
 - lib/webget.rb
 - lib/webget/version.rb
 - lib/webget/webcache.rb
+- lib/webget/webcache_disk-rewrite.rb
+- lib/webget/webcache_disk.rb
+- lib/webget/webcache_headers.rb
 - lib/webget/webget.rb
-homepage: https://github.com/rubycoco/webclient
+homepage: https://github.com/rubycocos/webclient
 licenses:
 - Public Domain
 metadata: {}
@@ -117,14 +106,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: webget gem - a web (go get) crawler incl. web cache