pluto 0.8.6 → 0.8.7

data/lib/pluto/cli/main.rb

@@ -285,9 +285,9 @@ command [:update, :up, :u] do |c|
 
       config = load_config( arg )
 
-      Pluto::Updater.new( opts, config ).run
+      Pluto::Updater.new( opts, config ).run( name )
     end
-    
+
     puts 'Done.'
   end
 end # command fetch
@@ -326,7 +326,7 @@ command [:build, :b] do |c|
 
       config = load_config( arg )
 
-      Pluto::Updater.new( opts, config ).run
+      Pluto::Updater.new( opts, config ).run( name )
       Pluto::Formatter.new( opts, config ).run( name )
     end
     

data/lib/pluto/fetcher.rb

@@ -16,12 +16,12 @@ class Fetcher
   def fetch_feed( url )
     response = @worker.get( url )
 
-    if debug?
+    ## if debug?
       puts "http status #{response.code} #{response.message}"
       
       puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
       puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
-    end
+    ## end
 
     xml = response.body
 
@@ -130,10 +130,10 @@ class Fetcher
       fetched:            feed_fetched
     }
     
-    if debug?
+    ## if debug?
       puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
       puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
-    end
+    ## end
 
     feed_rec.update_attributes!( feed_attribs )
 

data/lib/pluto/formatter.rb

@@ -16,13 +16,29 @@ class Formatter
 
   attr_reader :opts, :config, :site
 
+
   def run( arg )
+    ### remove - always use make( site_key )
     ## fix: change arg to planet_key or just key or similar
     #  todo: rename run to some less generic  - merge/build/etc. ??
     
+    site_key = arg
+    site_key = site_key.downcase.gsub('.ini','').gsub('.yml','')  # remove .ini|.yml extension if present
+
     manifest_name = opts.manifest
+    output_path   = opts.output_path
+
+    make_for(site_key, manifest_name, output_path )
+  end
+
+
+  def make_for( site_key, manifest_name, output_path )
+
+    ## fix: remove reference to opts
+    ##  - e.g. now still used for auto-installer
+
     manifest_name = manifest_name.downcase.gsub('.txt', '' )  # remove .txt if present
-    
+
     logger.debug "manifest=#{manifest_name}"
 
     # check for matching manifests
@@ -47,14 +63,8 @@ class Formatter
     end
 
     manifestsrc = manifests[0][1]
-    pakpath     = opts.output_path
-
-    name = arg
-
-    ## for now - use single site w/ key planet
-    ##-- fix!! allow multiple sites (planets)
+    pakpath     = output_path
 
-    site_key = 'planet'
     @site = Site.find_by_key( site_key )
     if @site.nil?
       puts "*** warn: no site with key '#{site_key}' found; using untitled site record"
@@ -62,7 +72,7 @@ class Formatter
       @site.title = 'Planet Untitled'
     end
 
-    Pakman::Templater.new.merge_pak( manifestsrc, pakpath, binding, name )
+    Pakman::Templater.new.merge_pak( manifestsrc, pakpath, binding, site_key )
   end
 
 end # class Formatter

data/lib/pluto/models/feed.rb

@@ -17,7 +17,7 @@ class Feed < ActiveRecord::Base
 
     # note: if not published, touched or built use hardcoded 1971-01-01 for now
     ## order( "coalesce(published,touched,built,'1971-01-01') desc" )
-    order( "coalesce(last_published,'1971-01-01') desc" )
+    order( "coalesce(feeds.last_published,'1971-01-01') desc" )
   end
 
   ##################################

data/lib/pluto/models/item.rb

@@ -24,7 +24,7 @@ class Item < ActiveRecord::Base
     #   coalesce - supported by sqlite (yes), postgres (yes)
 
     # note: if not published,touched or built_at use hardcoded 1971-01-01 for now
-    order( "coalesce(published,touched,'1971-01-01') desc" )
+    order( "coalesce(items.published,items.touched,'1971-01-01') desc" )
   end
 
   def published?()  read_attribute(:published).present?;  end

data/lib/pluto/models/utils.rb

@@ -23,14 +23,18 @@ class ItemCursor
         new_date = true
       end
 
-      if last_feed_id == item.feed.id
-        new_feed = false
-      else
+## note:
+#   new date also **always** starts new feed
+#  - e.g. used for grouping within day (follows planet planet convention)
+
+      if new_date || last_feed_id != item.feed.id 
         new_feed = true
+      else
+        new_feed = false
       end
 
       yield( item, new_date, new_feed )
-      
+
       last_published = item.published
       last_feed_id   = item.feed.id
     end

data/lib/pluto/refresher.rb

@@ -13,8 +13,8 @@ class Refresher
   def debug=(value)  @debug = value;   end
   def debug?()       @debug || false;  end
 
-  def update_feeds( opts={} )
 
+  def update_feeds( opts={} )  # update all feeds
     if debug?
       ## turn on logging for sql too
       ActiveRecord::Base.logger = Logger.new( STDOUT )
@@ -23,61 +23,83 @@ class Refresher
 
     # -- log update action
     Action.create!( title: 'update feeds' )
+    
+    feeds_fetched = Time.now
+    Site.all.each do |site|
+      site.update_attributes!( fetched: feeds_fetched )
+    end
+
+    Feed.all.each do |feed|
+      update_feed_worker( feed )
+    end
+  end
+
+
+  def update_feeds_for( site_key, opts={} )
+    if debug?
+      ## turn on logging for sql too
+      ActiveRecord::Base.logger = Logger.new( STDOUT )
+      @worker.debug = true   # also pass along worker debug flag if set
+    end
+
+    # -- log update action
+    Action.create!( title: "update feeds >#{site_key}<" )
 
     #####
     # -- update fetched  timestamps for all sites
     feeds_fetched = Time.now
-    Site.all.each do |site|
-      site.fetched = feeds_fetched
-      site.save!
+
+    site = Site.find_by_key!( site_key )
+    site.update_attributes!( fetched: feeds_fetched )
+
+    site.feeds.each do |feed|
+      update_feed_worker( feed )
     end
 
-    Feed.all.each do |feed_rec|
+  end # method update_feeds
 
-      feed = @worker.feed_by_rec_if_modified( feed_rec )
+private
+  def update_feed_worker( feed_rec )
+    feed = @worker.feed_by_rec_if_modified( feed_rec )
       
-      # on error or if http-not modified etc. skip update/processing
-      next  if feed.nil?
+    # on error or if http-not modified etc. skip update/processing
+    return  if feed.nil?
 
-      ## fix/todo: reload feed_red   - fetched date updated etc.
-      ##  check if needed for access to fetched date
+    ## fix/todo: reload feed_red   - fetched date updated etc.
+    ##  check if needed for access to fetched date
       
 
-      ## todo/check: move feed_rec update to the end (after item updates??)
+    ## todo/check: move feed_rec update to the end (after item updates??)
 
-      # update feed attribs e.g.
-      #    generator
-      #    published_at,built_at,touched_at,fetched_at
-      #    summary,title2
+    # update feed attribs e.g.
+    #    generator
+    #    published_at,built_at,touched_at,fetched_at
+    #    summary,title2
       
-      ## fix:
-      ## weird rss exception error on windows w/ dates
-      #  e.g. /lib/ruby/1.9.1/rss/rss.rb:37:in `w3cdtf': wrong number of arguments (1 for 0) (ArgumentError)
-      #
-      #  move to_datetime to feedutils!! if it works
-      ##   todo: move this comments to feedutils??
-
-
-      feed_rec.debug = debug? ? true : false    # pass along debug flag
-      ## fix/todo: pass debug flag as opts - debug: true|false !!!!!!
-      feed_rec.save_from_struct!( feed )  # todo: find a better name - why? why not??
-
-
-      #  update  cached value last published for item
-      last_item_rec = feed_rec.items.latest.limit(1).first  # note limit(1) will return relation/arrar - use first to get first element or nil from ary
-      if last_item_rec.present?
-        if last_item_rec.published?
-          feed_rec.update_attributes!( last_published: last_item_rec.published )
-        else # try touched
-          feed_rec.update_attributes!( last_published: last_item_rec.touched )
-        end
+    ## fix:
+    ## weird rss exception error on windows w/ dates
+    #  e.g. /lib/ruby/1.9.1/rss/rss.rb:37:in `w3cdtf': wrong number of arguments (1 for 0) (ArgumentError)
+    #
+    #  move to_datetime to feedutils!! if it works
+    ##   todo: move this comments to feedutils??
+
+
+    feed_rec.debug = debug? ? true : false    # pass along debug flag
+    ## fix/todo: pass debug flag as opts - debug: true|false !!!!!!
+    feed_rec.save_from_struct!( feed )  # todo: find a better name - why? why not??
+
+
+    #  update  cached value last published for item
+    last_item_rec = feed_rec.items.latest.limit(1).first  # note limit(1) will return relation/arrar - use first to get first element or nil from ary
+    if last_item_rec.present?
+      if last_item_rec.published?
+        feed_rec.update_attributes!( last_published: last_item_rec.published )
+      else # try touched
+        feed_rec.update_attributes!( last_published: last_item_rec.touched )
       end
+    end
+  end  # method update_feed_worker
 
-    end # each feed
-
-  end # method update_feeds
-
- 
 end # class Refresher
 
 end # module Pluto

data/lib/pluto/schema.rb

@@ -9,6 +9,12 @@ class CreateDb < ActiveRecord::Migration
       t.string   :key,       :null => false    # e.g. ruby, js, etc.
       t.datetime :fetched   #  last fetched/checked date -- make not null ??
 
+      ############
+      # filters (site-wide)
+      t.string   :includes  # regex
+      t.string   :excludes  # regex
+
+
       t.timestamps  # created_at, updated_at
     end
 

data/lib/pluto/subscriber.rb

@@ -11,15 +11,17 @@ class Subscriber
 
 
   def update_subscriptions( config, opts={} )
+    # !!!! -- depreciated API - remove - do NOT use anymore
+    puts "warn - [Pluto::Subscriber] depreciated API -- use update_subscriptions_for( site_key)"
+    update_subscriptions_for( 'planet', config, opts )  # default to planet site_key
+  end
 
+
+  def update_subscriptions_for( site_key, config, opts={} )
     site_attribs = {
-      title: config[ 'title' ] || config[ 'name' ]   # support either title or name
+      title: config['title'] || config['name']   # support either title or name
     }
-
-    ## for now - use single site w/ key planet
-    ##  -- fix!! allow multiple sites (planets)
-    
-    site_key = 'planet'
+ 
     site_rec = Site.find_by_key( site_key )
     if site_rec.nil?
       site_rec             = Site.new
@@ -27,12 +29,12 @@ class Subscriber
 
       ## use object_id: site.id and object_type: Site
       ## change - model/table/schema!!!
-      Action.create!( title: 'new site', object: site_attribs[ :title ] )
+      Action.create!( title: "new site >#{site_key}<", object: site_attribs[ :title ] )
     end
     site_rec.update_attributes!( site_attribs )
 
     # -- log update action
-    Action.create!( title: 'update subscriptions' )
+    Action.create!( title: "update subscriptions >#{site_key}<" )
 
     # clean out subscriptions and add again
     logger.debug "before site.subscriptions.delete_all - count: #{site_rec.subscriptions.count}"
@@ -44,7 +46,10 @@ class Subscriber
       ## todo: downcase key - why ??? why not???
 
       # skip "top-level" feed keys e.g. title, etc. or planet planet sections (e.g. planet,defaults)
-      next if ['title','title2','name','feeds','planet','defaults'].include?( key )   
+      next if ['title','title2','name',
+               'include','includes','exclude','excludes',
+               'feeds',
+               'planet','defaults'].include?( key )
 
       ### todo/check:
       ##   check value - must be hash
@@ -76,9 +81,9 @@ class Subscriber
         ## use object_id: feed.id and object_type: Feed
         ## change - model/table/schema!!!
         ## todo: add parent_action_id - why? why not?
-        Action.create!( title: 'new feed', object: feed_attribs[ :title ] )
+        Action.create!( title: "new feed >#{feed_key}<", object: feed_attribs[ :title ] )
       end
-      
+
       feed_rec.update_attributes!( feed_attribs )
 
       #  add subscription record

data/lib/pluto/updater.rb

@@ -11,14 +11,20 @@ class Updater
 
   attr_reader :opts, :config
 
-  def run
+  def run( arg )
+    arg = arg.downcase.gsub('.ini','').gsub('.yml','')  # remove file extension if present
+
+    update_for( arg )
+  end
+
+  def update_for( site_key )
     ###################
     # step 1) update subscriptions
     subscriber = Subscriber.new
 
     # pass along debug/verbose setting/switch
     subscriber.debug = true    if opts.verbose?
-    subscriber.update_subscriptions( config )
+    subscriber.update_subscriptions_for( site_key, config )
 
     ##############################
     # step 2) update feeds
@@ -26,7 +32,7 @@ class Updater
 
     # pass along debug/verbose setting/switch
     refresher.debug = true    if opts.verbose?
-    refresher.update_feeds
+    refresher.update_feeds_for( site_key )
   end # method run
 
 end # class Updater

data/lib/pluto/version.rb

@@ -1,4 +1,4 @@
 
 module Pluto
-  VERSION = '0.8.6'
+  VERSION = '0.8.7'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pluto
 version: !ruby/object:Gem::Version
-  version: 0.8.6
+  version: 0.8.7
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pakman
-  requirement: &75090920 !ruby/object:Gem::Requirement
+  requirement: &79324440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *75090920
+  version_requirements: *79324440
 - !ruby/object:Gem::Dependency
   name: fetcher
-  requirement: &75090620 !ruby/object:Gem::Requirement
+  requirement: &79324160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0.3'
   type: :runtime
   prerelease: false
-  version_requirements: *75090620
+  version_requirements: *79324160
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &75090350 !ruby/object:Gem::Requirement
+  requirement: &79323870 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0.6'
   type: :runtime
   prerelease: false
-  version_requirements: *75090350
+  version_requirements: *79323870
 - !ruby/object:Gem::Dependency
   name: feedutils
-  requirement: &75090050 !ruby/object:Gem::Requirement
+  requirement: &79323600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *75090050
+  version_requirements: *79323600
 - !ruby/object:Gem::Dependency
   name: props
-  requirement: &75089760 !ruby/object:Gem::Requirement
+  requirement: &79323280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *75089760
+  version_requirements: *79323280
 - !ruby/object:Gem::Dependency
   name: textutils
-  requirement: &75089460 !ruby/object:Gem::Requirement
+  requirement: &79323050 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.6.8
   type: :runtime
   prerelease: false
-  version_requirements: *75089460
+  version_requirements: *79323050
 - !ruby/object:Gem::Dependency
   name: gli
-  requirement: &75089160 !ruby/object:Gem::Requirement
+  requirement: &79322720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 2.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *75089160
+  version_requirements: *79322720
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &75088910 !ruby/object:Gem::Requirement
+  requirement: &79322460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +98,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *75088910
+  version_requirements: *79322460
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &75088620 !ruby/object:Gem::Requirement
+  requirement: &79322170 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,7 +109,7 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *75088620
+  version_requirements: *79322170
 description: pluto - Another Planet Generator (Lets You Build Web Pages from Published
   Web Feeds)
 email: feedreader@googlegroups.com