pluto-models 1.1.0

data/lib/pluto/models/item.rb

@@ -0,0 +1,73 @@
+module Pluto
+  module Models
+
+class Item < ActiveRecord::Base
+  self.table_name = 'items'
+
+  include Pluto::ActiveRecordMethods  # e.g. read_attribute_w_fallbacks
+
+  belongs_to :feed
+
+  ##################################
+  # attribute reader aliases
+  def name()        title;    end  # alias for title
+  def description() summary;  end  # alias for summary  -- also add descr shortcut??
+  def link()        url;      end  # alias for url
+
+  def self.latest
+    # note: order by first non-null datetime field
+    #   coalesce - supported by sqlite (yes), postgres (yes)
+
+    # note: if not published,touched or built_at use hardcoded 1971-01-01 for now
+    order( "coalesce(items.published,items.touched,'1971-01-01') desc" )
+  end
+
+  def published?()  read_attribute(:published).present?;  end
+
+  def published
+    ## todo/fix: use a new name - do NOT squeeze convenience lookup into existing
+    #    db backed attribute
+
+    read_attribute_w_fallbacks(
+      :published,
+      :touched       # try touched (aka updated RSS/ATOM)
+    )
+  end
+
+
+
+  def debug=(value)  @debug = value;   end
+  def debug?()       @debug || false;  end
+
+  def update_from_struct!( feed_rec, data )
+    ## check: new item/record?  not saved?  add guid
+    #   otherwise do not add guid  - why? why not?
+
+    item_attribs = {
+      guid:         data.guid,   # todo: only add for new records???
+      title:        data.title,
+      url:          data.url,
+      summary:      data.summary,
+      content:      data.content,
+      published:    data.published,
+      touched:      data.updated,
+      feed_id:      feed_rec.id,    # add feed_id fk_ref
+      fetched:      feed_rec.fetched
+    }
+
+    if debug?
+      puts "*** dump item_attribs w/ class types:"
+      item_attribs.each do |key,value|
+        next if [:summary,:content].include?( key )   # skip summary n content
+        puts "  #{key}: >#{value}< : #{value.class.name}"
+      end
+    end
+
+    update_attributes!( item_attribs )
+  end
+
+end  # class Item
+
+
+  end # module Models
+end # module Pluto

data/lib/pluto/models/site.rb

@@ -0,0 +1,19 @@
+module Pluto
+  module Models
+
+class Site < ActiveRecord::Base
+  self.table_name = 'sites'
+  
+  has_many :subscriptions
+  has_many :feeds, :through => :subscriptions
+  has_many :items, :through => :feeds
+
+  ##################################
+  # attribute reader aliases
+  def name()        title;    end  # alias for title
+  def fetched_at()  fetched;  end  # - legacy attrib reader -- remove!!!
+
+end
+
+  end # module Models
+end # module Pluto

data/lib/pluto/models/subscription.rb

@@ -0,0 +1,14 @@
+module Pluto
+  module Models
+
+
+class Subscription < ActiveRecord::Base
+  self.table_name = 'subscriptions'
+  
+  belongs_to :site
+  belongs_to :feed
+end
+
+
+  end # module Models
+end # module Pluto

data/lib/pluto/models/utils.rb

@@ -0,0 +1,47 @@
+module Pluto
+  module Models
+
+class ItemCursor
+
+  def initialize( items )
+    @items = items
+  end
+
+  def each
+    last_published = Time.local( 1971, 1, 1 )
+    last_feed_id      = -1     ## todo: use feed_key instead of id?? why? why not??
+    
+    @items.each do |item|
+
+      item_published = item.published  # cache published value ref
+
+      if last_published.year   == item_published.year  &&
+         last_published.month  == item_published.month &&
+         last_published.day    == item_published.day 
+        new_date = false
+      else
+        new_date = true
+      end
+
+## note:
+#   new date also **always** starts new feed
+#  - e.g. used for grouping within day (follows planet planet convention)
+
+      if new_date || last_feed_id != item.feed.id 
+        new_feed = true
+      else
+        new_feed = false
+      end
+
+      yield( item, new_date, new_feed )
+
+      last_published = item.published
+      last_feed_id   = item.feed.id
+    end
+  end # method each
+    
+end # class ItemCursor
+
+
+  end # module Models
+end # module Pluto

data/lib/pluto/refresher.rb

@@ -0,0 +1,130 @@
+module Pluto
+
+class Refresher
+
+  include LogUtils::Logging
+
+  include Models
+
+  def initialize
+    @worker  = Fetcher.new
+  end
+
+  def debug=(value)  @debug = value;   end
+  def debug?()       @debug || false;  end
+
+
+  def update_sites( opts={} )  # update all site configs
+    if debug?
+      ## turn on logging for sql too
+      ActiveRecord::Base.logger = Logger.new( STDOUT )
+      @worker.debug = true   # also pass along worker debug flag if set
+    end
+
+    # -- log update activity
+    Activity.create!( text: "update sites (#{Site.count})" )
+
+    #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
+
+    Site.order(:id).each do |site|
+      update_site_worker( site )  if site.url.present?  # note: only update if (source) url present
+    end
+  end
+
+
+  def update_feeds( opts={} )  # update all feeds
+    if debug?
+      ## turn on logging for sql too
+      ActiveRecord::Base.logger = Logger.new( STDOUT )
+      @worker.debug = true   # also pass along worker debug flag if set
+    end
+
+    # -- log update activity
+    Activity.create!( text: "update feeds (#{Feed.count})" )
+
+    #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
+
+    Feed.order(:id).each do |feed|
+      update_feed_worker( feed )
+    end
+  end
+
+
+  def update_feeds_for( site_key, opts={} )
+    if debug?
+      ## turn on logging for sql too
+      ActiveRecord::Base.logger = Logger.new( STDOUT )
+      @worker.debug = true   # also pass along worker debug flag if set
+    end
+
+    # -- log update activity
+    Activity.create!( text: "update feeds >#{site_key}<" )
+
+    site = Site.find_by_key!( site_key )
+
+    site.feeds.each do |feed|
+      update_feed_worker( feed )
+    end
+
+  end # method update_feeds
+
+
+private
+  def update_site_worker( site_rec )
+    site_config = @worker.site_by_rec_if_modified( site_rec )
+  
+    # on error or if http-not modified etc. skip update/processing
+    return  if site_config.nil?
+
+    subscriber = Subscriber.new
+    subscriber.debug = debug? ? true : false    # pass along debug flag
+
+    site_key = site_rec.key
+    subscriber.update_subscriptions_for( site_key, site_config )
+  end
+
+
+  def update_feed_worker( feed_rec )
+    feed = @worker.feed_by_rec_if_modified( feed_rec )
+      
+    # on error or if http-not modified etc. skip update/processing
+    return  if feed.nil?
+
+    ## fix/todo: reload feed_red   - fetched date updated etc.
+    ##  check if needed for access to fetched date
+      
+
+    ## todo/check: move feed_rec update to the end (after item updates??)
+
+    # update feed attribs e.g.
+    #    generator
+    #    published_at,built_at,touched_at,fetched_at
+    #    summary,title2
+      
+    ## fix:
+    ## weird rss exception error on windows w/ dates
+    #  e.g. /lib/ruby/1.9.1/rss/rss.rb:37:in `w3cdtf': wrong number of arguments (1 for 0) (ArgumentError)
+    #
+    #  move to_datetime to feedutils!! if it works
+    ##   todo: move this comments to feedutils??
+
+
+    feed_rec.debug = debug? ? true : false    # pass along debug flag
+    ## fix/todo: pass debug flag as opts - debug: true|false !!!!!!
+    feed_rec.save_from_struct!( feed )  # todo: find a better name - why? why not??
+
+
+    #  update  cached value last published for item
+    last_item_rec = feed_rec.items.latest.limit(1).first  # note limit(1) will return relation/arrar - use first to get first element or nil from ary
+    if last_item_rec.present?
+      if last_item_rec.published?
+        feed_rec.update_attributes!( last_published: last_item_rec.published )
+      else # try touched
+        feed_rec.update_attributes!( last_published: last_item_rec.touched )
+      end
+    end
+  end  # method update_feed_worker
+
+end # class Refresher
+
+end # module Pluto

data/lib/pluto/schema.rb

@@ -0,0 +1,139 @@
+
+module Pluto
+
+class CreateDb < ActiveRecord::Migration
+
+  def up
+    create_table :sites do |t|
+      t.string   :title,     :null => false    # e.g Planet Ruby, Planet JavaScript, etc.
+      t.string   :key,       :null => false    # e.g. ruby, js, etc.
+
+      ############
+      # filters (site-wide)
+      t.string   :includes  # regex
+      t.string   :excludes  # regex
+
+      ######################
+      # for auto-update of feed list/site config
+
+      t.string   :url    # source url for auto-update (optional)
+
+      ## note: make sure to use same fields for update check as feed 
+
+      t.datetime :fetched   #  last fetched/checked date -- make not null ??
+      t.integer  :http_code   # last http status code e.g. 200,404,etc.
+      t.string   :http_etag    # last http header etag
+      ## note: save last-modified header as text (not datetime) - pass through as is
+      t.string   :http_last_modified   # last http header last-modified - note: save header as plain text!!! pass along in next request as-is
+      t.string   :http_server    # last http server header if present
+
+      # note: do NOT store body content (that is, text) and md5 digest
+      #   use git! and github! commit will be http_etag!!
+
+      t.datetime :fetched    # last fetched/checked date
+
+      #############
+      # more fields
+
+      t.timestamps  # created_at, updated_at
+    end
+
+    create_table :subscriptions do |t|   # has_many join table (sites/feeds)
+      t.references :site, :null => false
+      t.references :feed, :null => false
+      t.timestamps
+    end
+
+    create_table :feeds do |t|
+      t.string  :title        # user supplied title
+      t.string  :auto_title   # "fallback" - auto(fill) title from feed
+
+      t.string  :title2        # user supplied title2
+      t.string  :auto_title2   # "fallback" - auto(fill) title2 from feed e.g. subtitle (atom)
+
+      t.string  :url          # user supplied site url
+      t.string  :auto_url     # "fallback" - auto(fill) url from feed
+
+      t.string  :feed_url       # user supplied feed url
+      t.string  :auto_feed_url  # "fallback" - auto discovery feed url from (site) url
+
+      t.text    :summary    # e.g. description (rss)
+
+      t.string  :generator   # feed generator (e.g. wordpress, etc.)  from feed
+      
+      t.datetime :published  # from feed published(atom)+ pubDate(rss)
+      t.datetime :built      # from feed lastBuiltDate(rss)
+      t.datetime :touched    # from feed updated(atom)
+
+      ### extras (move to array for custom fields or similar??)
+      t.string   :github   # github handle  (optional)
+      t.string   :twitter  # twitter handle (optional)
+      t.string   :avatar   # gravator or hackergotchi handle (optional)
+
+      ### add class/kind field e.g.
+      # - personal feed/blog/site, that is, individual author
+      # - team blog/site
+      # - org (anization) or com(pany blog/site)
+      # - newsfeed (composite)
+      # - other  (link blog?, podcast?) - why? why not??
+
+      ############
+      # filters
+      t.string   :includes  # regex
+      t.string   :excludes  # regex
+      # todo: add generic filter list e.g. t.string :filters  (comma,pipe or space separated method names?)
+
+      # -- our own (meta) fields
+      t.datetime :last_published # cache last (latest) published for items
+
+      t.string  :key,      :null => false
+      t.string  :format      # e.g. atom (1.0), rss 2.0, rss 0.7 etc.
+
+      t.integer  :http_code   # last http status code e.g. 200,404,etc.
+      t.string   :http_etag    # last http header etag
+      ## note: save last-modified header as text (not datetime) - pass through as is
+      t.string   :http_last_modified   # last http header last-modified - note: save header as plain text!!! pass along in next request as-is
+      t.string   :http_server    # last http server header if present
+
+      t.string   :md5       # md5 hash of body
+      t.text     :body      # last http response body (complete feed!)
+
+      t.datetime :fetched    # last fetched/checked date
+
+      t.timestamps   # created_at, updated_at
+    end
+
+
+    create_table :items do |t|
+      t.string   :guid
+      t.string   :url
+
+      ## note: title may contain more than 255 chars!! use text for sure!
+      ## e.g. Rails Girls blog has massive titles in feed
+      
+      t.text     :title   # todo: add some :null => false ??
+      t.text     :summary  # e.g. description (rss), summary (atom)
+      t.text     :content
+      
+      t.datetime :published   # from feed (published)  + pubDate(rss)
+      t.datetime :touched     # from feed updated (atom)
+
+      ## todo: add :last_updated_at ??  (NOTE: updated_at already take by auto-timestamps)
+      t.references :feed, :null => false
+
+      t.datetime :fetched   # last fetched/check date
+      t.timestamps   # created_at, updated_at
+
+      ## t.string   :author
+      ## todo: add author/authors, category/categories
+    end
+
+  end
+    
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+    
+end  # class CreateDb
+   
+end  # module Pluto

data/lib/pluto/subscriber.rb

@@ -0,0 +1,102 @@
+module Pluto
+
+class Subscriber
+
+  include LogUtils::Logging
+
+  include Models
+
+  def debug=(value)  @debug = value;   end
+  def debug?()       @debug || false;  end
+
+
+  def update_subscriptions( config, opts={} )
+    # !!!! -- depreciated API - remove - do NOT use anymore
+    puts "*** warn - [Pluto::Subscriber] depreciated API -- use update_subscriptions_for( site_key )"
+    update_subscriptions_for( 'planet', config, opts )  # default to planet site_key
+  end
+
+
+  def update_subscriptions_for( site_key, config, opts={} )
+    site_attribs = {
+      title:  config['title'] || config['name'],   # support either title or name
+      url:    config['source'] || config['url']    # support source or url   for source url for auto-update (optional)
+    }
+
+    logger.debug "site_attribs: #{site_attribs.inspect}"
+
+    site_rec = Site.find_by_key( site_key )
+    if site_rec.nil?
+      site_rec             = Site.new
+      site_attribs[ :key ] = site_key
+
+      ## use object_id: site.id and object_type: Site
+      ## change - model/table/schema!!!
+      Activity.create!( text: "new site >#{site_key}< - #{site_attribs[ :title ]}" )
+    end
+    site_rec.update_attributes!( site_attribs )
+
+    # -- log update activity
+    Activity.create!( text: "update subscriptions >#{site_key}<" )
+
+    # clean out subscriptions and add again
+    logger.debug "before site.subscriptions.delete_all - count: #{site_rec.subscriptions.count}"
+    site_rec.subscriptions.destroy_all  # note: use destroy_all NOT delete_all (delete_all tries by default only nullify)
+    logger.debug "after site.subscriptions.delete_all - count: #{site_rec.subscriptions.count}"
+
+    config.each do |key, value|
+
+      ## todo: downcase key - why ??? why not???
+
+      # skip "top-level" feed keys e.g. title, etc. or planet planet sections (e.g. planet,defaults)
+      next if ['title','title2','name',
+               'source', 'url',
+               'include','includes','exclude','excludes',
+               'feeds',
+               'planet','defaults'].include?( key )
+
+      ### todo/check:
+      ##   check value - must be hash
+      #     check if url or feed_url present
+      #      that is, check for required props/key-value pairs
+
+      feed_key   = key.to_s.dup
+      feed_hash  = value
+
+      # todo/fix: use title from feed?
+      #  e.g. fill up auto_title, auto_url, etc.
+
+      feed_attribs = {
+        feed_url: feed_hash[ 'feed' ]  || feed_hash[ 'feed_url' ],
+        url:      feed_hash[ 'link' ]  || feed_hash[ 'url' ],
+        title:    feed_hash[ 'title' ] || feed_hash[ 'name' ],
+        title2:   feed_hash[ 'title2' ],
+        includes: feed_hash[ 'includes' ] || feed_hash[ 'include' ],
+        excludes: feed_hash[ 'excludes' ] || feed_hash[ 'exclude' ]
+      }
+
+      puts "Updating feed subscription >#{feed_key}< - >#{feed_attribs[:feed_url]}<..."
+
+      feed_rec = Feed.find_by_key( feed_key )
+      if feed_rec.nil?
+        feed_rec             = Feed.new
+        feed_attribs[ :key ] = feed_key
+
+        ## use object_id: feed.id and object_type: Feed
+        ## change - model/table/schema!!!
+        ## todo: add parent_action_id - why? why not?
+        Activity.create!( text: "new feed >#{feed_key}< - #{feed_attribs[ :title ]}" )
+      end
+
+      feed_rec.update_attributes!( feed_attribs )
+
+      #  add subscription record
+      #   note: subscriptions get cleaned out on update first (see above)
+      site_rec.subscriptions.create!( feed_id: feed_rec.id )   
+    end
+
+  end # method update_subscriptions
+
+end # class Subscriber
+
+end # module Pluto