pluto-update 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 537a23fa41963fdb77dfea36a67d3971673b0ec4
-  data.tar.gz: ca620b5fb7c8e25e236095576919be7b0d48155a
+  metadata.gz: d631c3eb45ac43efb9e2d0186270a5ff6adc0e43
+  data.tar.gz: bb4774dcb0d97bc05b3f19f05956f608d94bf18d
 SHA512:
-  metadata.gz: e6d7563a0cc2e7b6a3e2081ef2b1c1c6e1e30c7ba115220efa4cbd7f7c1356b85ccb0e42e665a78d6663a92dc9d2aacc8237c456eafa449cf77a7bb93e324a26
-  data.tar.gz: b3fba603c49a729c325f0935457739a9d73c04d36d58482921999e63888ba8cf4df4f00e54865314c56909fabedfc19affe767a645e747767c537862bf469124
+  metadata.gz: 858c5e290c621b417ab5773e3c307daed4a0c0174d00ef88729ac9a235deafc5e0c42027eaaf86220017415b9538be04c6661a7f989fedd5c87c5143c97e2fe9
+  data.tar.gz: 228ffdf0626f7455c73bc108bdc4639c5f2ff42d468238d2d6dcd308a8e0df3aead6f0f2cecf0d374a36b378b3babba0353ad10f26d5e8ca81ef4d102117302b

data/README.md

@@ -34,12 +34,9 @@ title = Planet Ruby
 ```
 
 For more samples, see [`nytimes.ini`](https://github.com/feedreader/planets/blob/master/nytimes.ini),
-[`js.ini`](https://github.com/feedreader/planets/blob/master/js.ini),
-[`dart.ini`](https://github.com/feedreader/planets/blob/master/dart.ini),
-[`haskell.ini`](https://github.com/feedreader/planets/blob/master/haskell.ini),
-[`viennarb.ini`](https://github.com/feedreader/planets/blob/master/viennarb.ini),
-[`beer.ini`](https://github.com/feedreader/planets/blob/master/beer.ini),
-[`football.ini`](https://github.com/feedreader/planets/blob/master/football.ini).
+[`js.ini`](https://github.com/feedreader/planet-web/blob/master/js.ini),
+[`dart.ini`](https://github.com/feedreader/planet-web/blob/master/dart.ini),
+[`haskell.ini`](https://github.com/feedreader/planets/blob/master/haskell.ini).
 
 
 

data/Rakefile

@@ -18,8 +18,9 @@ Hoe.spec 'pluto-update' do
   self.history_file = 'HISTORY.md'
 
   self.extra_deps = [
-    ['pluto-models',  '>= 1.2.2'],
+    ['pluto-models',  '>= 1.3.2'],
     ['fetcher',       '>= 0.4.4'],
+    ['preproc',       '>= 0.1.0'],
   ]
 
   self.licenses = ['Public Domain']

data/lib/pluto/update.rb

@@ -5,7 +5,8 @@ require 'pluto/models'
 
 
 # more 3rd party gems
-require 'fetcher'
+require 'fetcher'    # fetch (text) documents
+require 'preproc'    # include preprocessor
 
 
 # our own code

data/lib/pluto/update/fetcher.rb

@@ -262,15 +262,30 @@ class Fetcher
     response = @worker.get( site_url )
     @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
 
+    site_fetched = Time.now
+
+    ###
+    # Note: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will be set to ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+    # thus, set/force encoding to utf-8
+    site_text = response.body.to_s
+    site_text = site_text.force_encoding( Encoding::UTF_8 )
+
     if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
-      puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
-      puts "no change; request returns not modified (304); skipping parsing site config"
-      return nil   # no updates available; nothing to do
-    end
 
-    site_fetched = Time.now
+      if site_text.index('@include')
+        ## note: if the site_text includes @include
+        ##  we must revalidate complete file hierachy(tree) for now
+        ### continue;
+        ##
+        ##  fix/todo: use ahead-of-time preprocessor ?? in the future to simplify???
+      else
+        puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+        puts "no change; request returns not modified (304); skipping parsing site config"
+        return nil   # no updates available; nothing to do
+      end
 
-    if response.code != '200'   # note Net::HTTP response.code is a string in ruby
+    elsif response.code != '200'   # note Net::HTTP response.code is a string in ruby
 
       puts "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
 
@@ -279,9 +294,7 @@ class Fetcher
         http_server:        response.header[ 'server' ],
         http_etag:          nil,
         http_last_modified: nil,
-        body:               nil,
-        md5:                nil,
-        fetched:            feed_fetched 
+        fetched:            site_fetched 
       }
       site_rec.update_attributes!( site_attribs )
       
@@ -289,27 +302,12 @@ class Fetcher
       Activity.create!( text: "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}" )
 
       return nil  #  sorry; no feed for parsing available
+    else
+      # assume 200; continue w/ processing
     end
 
     puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
 
-    site_text = response.body
-    
-    ###
-    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will mostly be ASCII
-    # - try to change encoding to UTF-8 ourselves
-    logger.debug "site_text.encoding.name (before): #{site_text.encoding.name}"
-
-    #####
-    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-
-    ## NB:
-    # for now "hardcoded" to utf8 - what else can we do?
-    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    site_text = site_text.force_encoding( Encoding::UTF_8 )
-    logger.debug "site_text.encoding.name (after): #{site_text.encoding.name}"
-
     site_attribs = {
       http_code:          response.code.to_i,
       http_server:        response.header[ 'server' ],
@@ -326,6 +324,23 @@ class Fetcher
 
     site_rec.update_attributes!( site_attribs )
 
+
+    #################
+    ### fix: add support for http_etag cache etc.  - how??
+    ###
+    ###   use from_text( text, base: base )  !!!!!!!!
+    ###    do NOT reissue first request
+    ##
+    ##  fix: use special case/method for update_with_includes!!!
+    ##    keep it simple w/o includes (do NOT mix in one method)
+    ##  split into two methods!!!
+
+    ## retry w/ preprocesser
+    ## refetch if @include found w/ all includes included
+    if site_text.index('@include')
+      site_text = InclPreproc.from_url( site_url ).read
+    end
+
     ## logger.debug "site_text:"
     ## logger.debug site_text[ 0..300 ] # get first 300 chars
 

data/lib/pluto/update/refresher.rb

@@ -24,14 +24,16 @@ class Refresher
       @worker.debug = true   # also pass along worker debug flag if set
     end
 
-    # -- log update activity
-    Activity.create!( text: "update sites (#{Site.count})" )
+    start_time = Time.now
+    Activity.create!( text: "start update sites (#{Site.count})" )
 
     #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
-
     Site.order(:id).each do |site|
       update_site_worker( site )  if site.url.present?  # note: only update if (source) url present
     end
+
+    total_secs = Time.now - start_time
+    Activity.create!( text: "done update sites (#{Site.count}) in #{total_secs}s" )
   end
 
 
@@ -42,14 +44,17 @@ class Refresher
       @worker.debug = true   # also pass along worker debug flag if set
     end
 
-    # -- log update activity
-    Activity.create!( text: "update feeds (#{Feed.count})" )
+    start_time = Time.now
+    Activity.create!( text: "start update feeds (#{Feed.count})" )
 
     #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
-
     Feed.order(:id).each do |feed|
       update_feed_worker( feed )
+      ### todo/fix: add catch exception in loop and log to activity log and continue w/ next feed
     end
+
+    total_secs = Time.now - start_time
+    Activity.create!( text: "done update feeds (#{Site.count}) in #{total_secs}s" )
   end
 
 

data/lib/pluto/update/version.rb

@@ -4,7 +4,7 @@
 module PlutoUpdate
 
   MAJOR = 1
-  MINOR = 3
+  MINOR = 4
   PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pluto-update
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-25 00:00:00.000000000 Z
+date: 2015-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pluto-models
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: 1.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: 1.3.2
 - !ruby/object:Gem::Dependency
   name: fetcher
   requirement: !ruby/object:Gem::Requirement
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.4
+- !ruby/object:Gem::Dependency
+  name: preproc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement