webget 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de0060acabe176d1a11a4f2e36e8bee5090083c4907b75d9b22e9cbb9c7b22e2
-  data.tar.gz: fb64c2b1294932b00fa401af006cd27b263b255789bad75e4ef62bec5598fd74
+  metadata.gz: 6912004e235fc4a3aad299490d04d9ab9d44ea5f98005a348aa7a456349cfbcb
+  data.tar.gz: f2b8bcda71c738557c76957eb4e6e78ee4af52ec9311e5c18a47f27baf2b45df
 SHA512:
-  metadata.gz: d1f1653c68729e7d609c3c848e0146de9668cd2932ddab2e794ab361d3f786603a10c7586bcf0029de4955255803987000b2fb35b0d1e14cfa35d0582f919be7
-  data.tar.gz: c6cff08b2f683bb5b8e39607735250f954115244be3e19dfcdeb29fd2d0a0f0a27be1fed5a2add9a1b75a9cbe81dbafea1e80f06b0a1bdd93d4bd3bd12d5a9a1
+  metadata.gz: 40d7bf5f844e00b67554455f1c191a7f189169ad075bffd0b8f8888f856128ed1c1ece29bf1a5932c7d22dc841b9f3b2c2f8ce9640fbed70ba4952c0cd7aa2f7
+  data.tar.gz: 5e4d940c50db3f43bb10160a2019eaa7dfc8d56743bf8d5aa1ad8130db089a971d8eb18ab7913398a5606a9347288734d34b5e5ad19bce63b903de899bcd3f3a

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-### 0.3.1
+### 0.3.2
 
 ### 0.0.1 / 2020-10-04
 

data/Rakefile

@@ -26,7 +26,7 @@ Hoe.spec 'webget' do
   self.licenses = ['Public Domain']
 
   self.spec_extras = {
-   required_ruby_version: '>= 2.2.2'
+   required_ruby_version: '>= 3.1.0'
   }
 
 end

data/lib/webget/version.rb

@@ -2,7 +2,7 @@
 class Webget
   MAJOR = 0    ## todo: namespace inside version or something - why? why not??
   MINOR = 3
-  PATCH = 1
+  PATCH = 2
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/lib/webget/webcache.rb

@@ -2,6 +2,69 @@
 
 module Webcache
 
+
+  class Headers # nested class for convenience access to (meta) headers
+
+    def self.parse( txt )
+      data = {}
+      txt.each_line do |line|
+         line = line.strip
+         next  if line.empty? || line.start_with?( '#' )
+
+         key, value = line.split( ':', 2 )  ## split on first colon
+         ## always downcase keys for now
+         ##  and strip value from leading and trailing spaces
+         ##
+         ##  todo/fix: deal with possible duplicate header keys!!
+         ##   if duplicate do NOT replease, add with leading ", " comma-separated!!!
+         ##
+         ##  check if multi-line headers are possible!!!
+         data[ key.strip.downcase ] = value.strip
+      end
+      new( data )
+    end
+
+
+
+    def initialize( data )
+      @data = data
+    end
+
+    def to_h() @data; end
+    def [](key) @data[key];; end
+
+    def each( &blk )
+      @data.each do |key, value|
+        blk.call( key, value )
+      end
+    end
+
+
+    def date
+       ## return date header
+       ##  parses the time as RFC 1123 date of HTTP-date defined by RFC 2616:
+       ##    day-of-week, DD month-name CCYY hh:mm:ss GMT
+       ##   !!! Note that the result is always UTC (GMT). !!!
+       ##   e.g. Sun, 19 May 2024 15:15:34 GMT
+       ##        Mon, 10 Jun 2024 15:58:16 GMT
+       @date ||= Time.httpdate( @data['date'] )
+       @date
+    end
+
+    ## default to 12h (60secs*60min*12h)
+    def expired?( expires_in_date=Time.now.utc-60*60*12 )
+      ## pp expires_in_date
+      expires_in_date > date
+    end
+
+    ## add convenience helpers - why? why not?
+    def expired_in_12h?() expired?( Time.now.utc-60*60*12 ); end
+    def expired_in_24h?() expired?( Time.now.utc-60*60*24 ); end
+    alias_method :expired_in_1d?, :expired_in_24h?
+  end # class Headers
+
+
+
   #####
   # copied from props gem, see Env.home
   #    - https://github.com/rubycoco/props/blob/master/props/lib/props/env.rb
@@ -73,6 +136,26 @@ module Webcache
  def self.read_json( url )  cache.read_json( url ); end
  def self.read_csv( url )   cache.read_csv( url );  end
 
+#### new - read (cached) meta data
+##    todo/check - find a better/different name - why? why not?
+##       e.g. read_headers or simply meta or headers or such
+  def self.read_meta( url ) cache.read_meta( url ); end
+
+  ## add convenience expire (shortcut) helpers
+  def self.expired?( url, expires_in: Time.now.utc-60*60*12 )
+     if cached?( url )
+       meta = read_meta( url )
+       meta.expired?( expires_in )
+     else
+       true  # note - not in cache; expired by default
+     end
+  end
+  def self.expired_in_12h?( url ) expired?( url, expires_in: Time.now.utc-60*60*12 ); end
+  def self.expired_in_24h?( url ) expired?( url, expires_in: Time.now.utc-60*60*24 ); end
+  class << self
+    alias_method :expired_in_1d?, :expired_in_24h?
+  end
+
 
 
 class DiskCache
@@ -103,6 +186,15 @@ class DiskCache
   end
 
 
+  def read_meta( url )
+    body_path = "#{Webcache.root}/#{url_to_path( url )}"
+    meta_path = "#{body_path}.meta.txt"
+    txt = File.open( meta_path, 'r:utf-8' ) {|f| f.read }
+    data = Headers.parse( txt )
+    data
+  end
+
+
   ## add more save / put / etc. aliases - why? why not?
   ##  rename to record_html - why? why not?
   def record( url, response,
@@ -177,7 +269,19 @@ class DiskCache
 
     ### special "prettify" rule for weltfussball
     ##   /eng-league-one-2019-2020/  => /eng-league-one-2019-2020.html
-    if host_dir.index( 'weltfussball.de' ) ||
+
+### todo/fix - move rules downstream to user - why? why not?
+
+    if host_dir.index( 'uefa.com' ) ||
+       host_dir.index( 'kicker.de' ) ||
+       host_dir.index( 'kicekr.at' )
+      if req_path.end_with?( '/' )
+        req_path = "#{req_path[0..-2]}.html"
+      else
+        puts "ERROR: expected request_uri for >#{host_dir}< ending with '/'; got: >#{req_path}<"
+        exit 1
+      end
+    elsif host_dir.index( 'weltfussball.de' ) ||
        host_dir.index( 'worldfootball.net' )
           if req_path.end_with?( '/' )
              req_path = "#{req_path[0..-2]}.html"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webget
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-04 00:00:00.000000000 Z
+date: 2024-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: webclient
@@ -117,7 +117,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="