muck-raker 0.1.15 → 0.1.16

data/README.rdoc

@@ -1,13 +1,20 @@
-Recommender
-===========
-The recommender adds the ability to aggregate, search and analyze data from many different web sources.
+= Muck Raker
+Muck Raker adds functionality that aggregates, searches and analyzes data from many different web sources.
 
-Installation
-===========
-Install gem dependencies:
+== Installation
 
+Be sure to add github as a gem source before proceeding as acts-as-taggable-on, mislav-will_paginate (and many other gems) live there.
+  gem sources -a http://gems.github.com
+
+Install muck raker:
+  sudo gem install muck-raker
+  
+Installing muck raker should install all of it's dependancies.  You can also install them manually:
+  sudo gem install httparty
   sudo gem install mbleigh-acts-as-taggable-on
   sudo gem install mislav-will_paginate
+  sudo gem install muck-feedbag
+  sudo gem install pauldix-feedzirra
 
 After install acts-as-taggable-on be sure to include the helper in application_helper.rb:
 
@@ -20,8 +27,14 @@ Add rake tasks to your Rakefile
 
   require 'muck_raker/tasks'
 
-Example
-=======
+== Example
+
+
+== Development
+
+=== Testing
+You will need to have mysql setup to run the migrations for the test database.  Muck raker has a few customizations which require mysql.
+
 
 
 Copyright (c) 2009 Justin Ball and Joel Duffin, released under the MIT license

data/Rakefile

@@ -36,6 +36,8 @@ desc 'Translate this gem'
 task :translate do
   file = File.join(File.dirname(__FILE__), 'locales', 'en.yml')
   system("babelphish -o -y #{file}")
+  path = File.join(File.dirname(__FILE__), 'app', 'views', 'raker_mailer')
+  system("babelphish -o -h #{path} -l en")
 end
 
 desc 'Generate documentation for the recommender plugin.'
@@ -57,6 +59,10 @@ begin
     gemspec.description = "The aggregation and recommendation engine for the muck system."
     gemspec.authors = ["Joel Duffin Justin Ball"]
     gemspec.rubyforge_project = 'muck-raker'
+    gemspec.add_dependency "mbleigh-acts-as-taggable-on"
+    gemspec.add_dependency "mislav-will_paginate"
+    gemspec.add_dependency "httparty"
+    gemspec.add_dependency "jbasdf-feedbag"
     gemspec.add_dependency "muck-engine"
     gemspec.add_dependency "muck-users"
     gemspec.add_dependency "muck-solr"

data/VERSION

@@ -1 +1 @@
-0.1.15
+0.1.16

data/app/controllers/admin/{recommender → muck}/feeds_controller.rb

@@ -1,6 +1,6 @@
-class Admin::Recommender::FeedsController < ApplicationController
+class Admin::Muck::FeedsController < ApplicationController
 
-  layout "default"
+  unloadable
 
   before_filter :login_required, :only => [:index, :edit, :ban, :unban, :destroy, :update, :new]
 

data/app/controllers/{recommender → muck}/entries_controller.rb

@@ -1,5 +1,7 @@
-class Recommender::EntriesController < ApplicationController
-
+class Muck::EntriesController < ApplicationController
+  
+  unloadable
+  
   def initialize
     @no_index = true
   end
@@ -35,19 +37,20 @@ class Recommender::EntriesController < ApplicationController
   end
   
   def show
-    @page_title = "Related Resources"
     @entry = Entry.find(params[:id], :include => :feed)
     if @entry.nil?
       render_text "Unable to find the specified document"
       return
     end
-    @entry_title = @entry.title + " (" + @entry.feed.short_title + ")"
+    @entry_title = @entry.title + ' (' + @entry.feed.short_title + ')'
+    @page_title = @entry_title + ' - ' + I18n.t('muck.raker.related_resources_title')
 #    I18n.locale = @entry.language[0..1]
     @limit = params[:limit] ? params[:limit].to_i : 20
     @limit = 40 if @limit > 40
 
     respond_to do |format|
-      format.html { 
+      format.html {
+      @recommendations = @entry.recommendations(@limit, 'relevance', params[:details] == 'true')
       if params[:details] == 'true'
         render :template => "entries/details"
       else

data/app/controllers/muck/feed_previews_controller.rb

@@ -0,0 +1,33 @@
+class Muck::FeedPreviewsController < ApplicationController
+
+  unloadable
+
+  def new
+    respond_to do |format|
+      format.pjs do
+        render_as_html do
+          render :template => 'feed_previews/new', :layout => false
+        end
+      end
+      format.html { render :template => 'feed_previews/new' }
+    end
+  end
+
+  def select_feeds
+    @feed = Feed.new(params[:feed])
+    @feeds = Feed.gather_information(@feed.uri)
+    respond_to do |format|
+      format.pjs do
+        render_as_html do
+          render :template => 'feed_previews/select_feeds', :layout => false
+        end
+      end
+      format.html { render :template => 'feed_previews/select_feeds' }
+    end
+  end
+  
+  def create
+    
+  end
+
+end

data/app/controllers/muck/feeds_controller.rb

@@ -0,0 +1,128 @@
+class Muck::FeedsController < ApplicationController
+
+  unloadable
+  
+  before_filter :get_parent
+  
+  def index
+    @feeds = Feed.valid.by_title.paginate(:page => @page, :per_page => @per_page)
+    respond_to do |format|
+      format.html { render :template => 'feeds/index' }
+      format.xml  { render :xml => @feeds.to_xml }
+    end
+  end
+
+  # pass layout=popup to remove most of the chrome
+  def show
+    @feed = Feed.find(params[:id])
+    @entries = @feed.entries
+    respond_to do |format|
+      format.html { render :template => 'feeds/show', :layout => params[:layout] || true  }
+      format.pjs { debugger; render :template => 'feeds/show', :layout => false }
+      format.json { render :json => @feed.as_json }
+    end
+  end
+
+  def new
+    respond_to do |format|
+      format.html { render :template => 'feeds/new', :layout => 'popup' }
+    end
+  end
+
+  def new_extended
+    respond_to do |format|
+      format.html { render :template => 'feeds/new_extended', :layout => 'popup' }
+    end
+  end
+
+  def create
+    @feed = Feed.new(params[:feed])
+    @feed.contributor = current_user # record the user that submitted the feed for auditing purposes
+    @feed.harvested_from_display_uri = @feed.display_uri
+    
+    # setup the feed to be harvested
+    @feed.entries_count = 0
+    @feed.last_requested_at = 4.weeks.ago
+    @feed.last_harvested_at = 4.weeks.ago
+
+    # associate the parent if present
+    @parent.feeds << @feed if @parent
+    
+    after_create_response(@feed.save)
+  end
+
+
+  def update
+    @feed = Feed.find(params[:id])
+    after_update_response(@feed.update_attributes(params[:feed]))
+  end
+
+  def destroy
+    @feed = Feed.find(params[:id])
+    @feed.destroy
+    after_destroy_response
+  end
+  
+  protected
+
+    def get_parent
+      if !params[:parent_type] || !params[:parent_id]
+        return
+      end
+      @klass = params[:parent_type].to_s.constantize
+      @parent = @klass.find(params[:parent_id])
+      unless has_permission_to_add_feed(current_user, @parent)
+        permission_denied
+      end
+    end
+  
+    def has_permission_to_add_feed(user, parent)
+      user == parent || parent.can_add_feed?(user)
+    end
+    
+    # Handles render and redirect after success or failure of the
+    # create action.  Override to perform a different action
+    def after_create_response(success)
+      if success
+        flash[:notice] = t('muck.raker.feed_successfully_created')
+        respond_to do |format|
+          format.html { redirect_to feed_path(@feed) }
+          format.pjs { render :template => 'feeds/show', :layout => false }
+          format.json { render :json => @feed.as_json }
+          format.xml  { head :created, :location => feed_url(@feed) }
+        end
+      else
+        respond_to do |format|
+          format.html { render :template => "feeds/new" }
+          format.pjs { render :template => "feeds/new", :layout => false }
+          format.json { render :json => @feed.as_json }
+          format.xml  { render :xml => @feed.errors.to_xml }
+        end
+      end
+    end
+  
+    # Handles render and redirect after success or failure of the
+    # update action.  Override to perform a different action
+    def after_update_response(success)
+      respond_to do |format|
+        if success
+          flash[:notice] = t('muck.raker.feed_successfully_updated')
+          format.html { redirect_to feed_path(@feed) }
+          format.xml  { head :ok }
+        else
+          format.html { render :template => "feeds/edit" }
+          format.xml  { render :xml => @feed.errors.to_xml }
+        end
+      end
+    end
+    
+    # Handles render and redirect after the delete action.
+    # Override to perform a different action
+    def after_destroy_response
+      respond_to do |format|
+        format.html { redirect_to feeds_path }
+        format.xml  { head :ok }
+      end
+    end
+  
+end

data/app/controllers/{recommender → muck}/recommendations_controller.rb

@@ -1,5 +1,7 @@
-class Recommender::RecommendationsController < ApplicationController
+class Muck::RecommendationsController < ApplicationController
   
+  unloadable
+    
   # GET /recommendations
   # GET /recommendations.xml
   def index
@@ -27,6 +29,7 @@ class Recommender::RecommendationsController < ApplicationController
         order = params[:order] || "mixed"
         redirect_to "/documents/" + @document.id.to_s + "?limit=" + @limit.to_s + "&order=" + order + "&details=" + @details.to_s if !@document.nil?
         render(:template => '/recommendations/document_not_found.html.erb', :layout => false) if @document.nil?
+        @recommendations = @entry.recommendations(@limit, params[:order] || "relevance")
       } 
       format.xml  { 
         render(:template => '/recommendations/index.xml.builder', :layout => false) 

data/app/controllers/{recommender → muck}/search_controller.rb

@@ -1,5 +1,7 @@
-class Recommender::SearchController < ApplicationController
+class Muck::SearchController < ApplicationController
 
+  unloadable
+  
   def initialize
     @limit = 50
     @no_index = true

data/app/helpers/muck_raker_feeds_helper.rb

@@ -1,3 +1,27 @@
 module MuckRakerFeedsHelper
   
+  def add_feed(parent = nil)
+    render :partial => 'parts/add_feed', :locals => {:parent => parent}
+  end
+  
+  def add_single_feed(parent = nil)
+    render :partial => 'parts/add_single_feed', :locals => {:parent => parent}
+  end
+  
+  def add_extended_feed(parent = nil)
+    render :partial => 'parts/add_extended_feed', :locals => {:parent => parent}
+  end
+
+  def new_feed_path_with_parent(parent)
+    if parent
+      feeds_path(make_parent_params(parent))
+    else
+      feeds_path
+    end
+  end
+    
+  def make_parent_params(parent)
+    { :parent_id => parent.id, :parent_type => parent.class.to_s }
+  end
+  
 end

data/app/models/attention.rb

@@ -1,2 +1,14 @@
+# == Schema Information
+#
+# Table name: attentions
+#
+#  id                 :integer(4)      not null, primary key
+#  attentionable_id   :integer(4)
+#  attentionable_type :string(255)
+#  entry_id           :integer(4)
+#  action_type        :string(255)
+#  weight             :float
+#
+
 class Attention < ActiveRecord::Base
-end
+end

data/app/models/click.rb

@@ -1,2 +1,14 @@
+# == Schema Information
+#
+# Table name: clicks
+#
+#  id                :integer(4)      not null, primary key
+#  recommendation_id :integer(4)
+#  when              :datetime        not null
+#  referrer          :string(2083)
+#  requester         :string(255)
+#  user_agent        :string(2083)
+#
+
 class Click < ActiveRecord::Base
 end

data/app/models/entry.rb

@@ -1,11 +1,39 @@
+# == Schema Information
+#
+# Table name: entries
+#
+#  id                      :integer(4)      not null, primary key
+#  feed_id                 :integer(4)      not null
+#  permalink               :string(2083)    default(""), not null
+#  author                  :string(2083)
+#  title                   :text            default(""), not null
+#  description             :text
+#  content                 :text
+#  unique_content          :boolean(1)
+#  published_at            :datetime        not null
+#  entry_updated_at        :datetime
+#  harvested_at            :datetime
+#  oai_identifier          :string(2083)
+#  language_id             :integer(4)
+#  direct_link             :string(2083)
+#  indexed_at              :datetime        default(Fri Jan 01 01:01:01 UTC 1971), not null
+#  relevance_calculated_at :datetime        default(Fri Jan 01 01:01:01 UTC 1971), not null
+#  popular                 :text
+#  relevant                :text
+#  other                   :text
+#  grain_size              :string(255)     default("unknown")
+#
+
 class Entry < ActiveRecord::Base
+  
   belongs_to :feed
+  belongs_to :language
   
   acts_as_taggable
   
   @@default_time_on_page = 60.0
   
-  acts_as_solr({:if => false, :fields => [{:feed_id => :integer}, {:grain_size => :string}]}, {:type_field => :type_s})
+ # acts_as_solr({:if => false, :fields => [{:feed_id => :integer}, {:grain_size => :string}]}, {:type_field => :type_s})
 
   def self.top_tags(tags = nil)
     
@@ -27,8 +55,21 @@ class Entry < ActiveRecord::Base
   end
   
   def recommendations(limit = 20, order = "relevance", details = false, omit_feeds = nil)
-    sql = "SELECT recommendations.id, dest_entry_id, permalink, entries.title, relevance_calculated_at, relevance, clicks, avg_time_at_dest AS avg_time_on_target, direct_link, feeds.short_title AS collection "
-    sql << ", entries.description, author, published_at " if details == true
+    sql = "SELECT recommendations.id, dest_entry_id, entries.permalink, entries.title, entries.description, entries.direct_link, feeds.short_title AS collection "
+    sql << ", relevance_calculated_at, relevance, clicks, avg_time_at_dest AS avg_time_on_target, author, published_at " if details == true
+    sql << "FROM recommendations "
+    sql << "INNER JOIN entries ON recommendations.dest_entry_id = entries.id "
+    sql << "INNER JOIN feeds ON entries.feed_id = feeds.id "
+    sql << "WHERE recommendations.entry_id = ? "
+    sql << ("AND entries.feed_id NOT IN (" + omit_feeds.gsub(/[^0-9,]/,'') + ") ") if omit_feeds != nil
+    sql << "ORDER BY " + order + " DESC "
+    sql << "LIMIT " + limit.to_s
+    Entry.find_by_sql([sql,self.id])
+  end
+
+  def recommendation_entries(limit = 20, order = "relevance", details = false, omit_feeds = nil)
+    sql = "SELECT recommendations.dest_entry_id AS id, entries.permalink, entries.title, entries.description, entries.direct_link, feeds.short_title AS collection "
+    sql << ", relevance_calculated_at, relevance, clicks, avg_time_at_dest AS avg_time_on_target, author, published_at " if details == true
     sql << "FROM recommendations "
     sql << "INNER JOIN entries ON recommendations.dest_entry_id = entries.id "
     sql << "INNER JOIN feeds ON entries.feed_id = feeds.id "
@@ -46,22 +87,14 @@ class Entry < ActiveRecord::Base
       
       # if they spend longer than two minutes on a page, we don't infer anything
       if time_on_page > 5 and time_on_page < 120
-#        puts "==============================old: " + session[:last_clicked_recommendation_uri]
-#        puts "==============================new: " + normalized_uri(uri)
         if normalized_uri(uri) != session[:last_clicked_recommendation_uri] 
           recommendation = Recommendation.find(recommendation_id)
           entry = Entry.find(recommendation.entry_id)
-#          puts "clicks ================= " + recommendation.clicks.to_s
-#          puts "old avg ================= " + recommendation.avg_time_at_dest.to_s
           new_avg = (recommendation.avg_time_at_dest*recommendation.clicks - @@default_time_on_page + time_on_page)/recommendation.clicks
           recommendation.avg_time_at_dest = new_avg
-#          puts "time on page ================= " + time_on_page.to_s
-#          puts "new average ==================== " + new_avg.to_s
           recommendation.save!
           entry.rank_recommendations
           session[:last_clicked_recommendation] = nil
-#        else
-#          puts "same!!!!!!!!!!!!!!!!!!!!!!!"
         end
       else
         session[:last_clicked_recommendation] = nil if time_on_page > 5
@@ -196,6 +229,7 @@ class Entry < ActiveRecord::Base
   end
   
   def rank_recommendations
+    return
     # get recommendations for the entry from the recommendations table
     recs = self.recommendations
     

data/app/models/feed.rb

@@ -1,20 +1,70 @@
-# require 'httparty'
-
+# == Schema Information
+#
+# Table name: feeds
+#
+#  id                         :integer(4)      not null, primary key
+#  uri                        :string(2083)
+#  display_uri                :string(2083)
+#  title                      :string(1000)
+#  short_title                :string(100)
+#  description                :text
+#  tag_filter                 :string(1000)
+#  top_tags                   :text
+#  priority                   :integer(4)      default(10)
+#  status                     :integer(4)      default(1)
+#  last_requested_at          :datetime
+#  last_harvested_at          :datetime
+#  harvest_interval           :integer(4)      default(86400)
+#  failed_requests            :integer(4)      default(0)
+#  error_message              :text
+#  service_id                 :integer(4)      default(0)
+#  login                      :string(255)
+#  password                   :string(255)
+#  created_at                 :datetime
+#  updated_at                 :datetime
+#  entries_changed_at         :datetime
+#  harvested_from_display_uri :string(2083)
+#  harvested_from_title       :string(1000)
+#  harvested_from_short_title :string(100)
+#  entries_count              :integer(4)
+#  default_language_id        :integer(4)      default(0)
+#  default_grain_size         :string(255)     default("unknown")
+#  contributor_id             :integer(4)
+#
 class Feed < ActiveRecord::Base
   
-  # include HTTParty
-  # format :xml
+  require 'httparty'
+  
+  include HTTParty
+  format :xml
   
+  validates_presence_of :uri
+
+  has_many :feed_parents
   has_many :entries
   belongs_to :contributor, :class_name => 'User', :foreign_key => 'contributor_id'
+  belongs_to :default_language, :class_name => 'Language', :foreign_key => 'default_language_id'
+
+  named_scope :banned, :conditions => "status < 0"
+  named_scope :valid, :conditions => "status >= 0"
+  named_scope :by_title, :order => "title ASC"
+  named_scope :recent, lambda { { :conditions => ['created_at > ?', 1.week.ago] } }
+  named_scope :by_newest, :order => "created_at DESC"
   
   # named_scope :ready_to_harvest, lambda { |*args| { :conditions => [ "feeds.last_harvested_at < ?", args.first || 1.day.ago.end_of_day ] } }
-                                                                        
+  
+  def after_create
+    if GlobalConfig.inform_admin_of_global_feed && self.feed_parents.empty?
+      RakerMailer.deliver_notification_feed_added(self) # Global feed.  Email the admin to let them know a feed has been added
+    end    
+  end
+
+  # harvest_interval is stored in seconds
   def refresh_interval_hours
     if self.harvest_interval
-      self.harvest_interval.split(':')[0]
+      self.harvest_interval/3600
     else
-        "168"
+      168 # TODO what's this?
     end
   end
   
@@ -26,5 +76,29 @@ class Feed < ActiveRecord::Base
   #   feed = Feed.first # .ready_to_harvest.first
   #   get(feed.uri)
   # end
+
+  # Gathers all available feed uris from the given uri and parses them into
+  # feed objects
+  def self.gather_information(uri)
+    feeds = []
+    @available_feeds = discover_feeds(uri)
+    @available_feeds.each do |feed|
+      feeds << make_feed(Feedzirra::Feed.fetch_and_parse(feed.url))
+    end
+    feeds
+  end
+  
+  # Turns a feed from feedzirra into a muck feed
+  def self.make_feed(feed)
+    Feed.new(:short_title => feed.title,
+             :title => feed.title,
+             :display_uri => feed.url,
+             :uri => feed.feed_url)
+  end
+  
+  # Looks for feeds from a given url
+  def self.discover_feeds(uri)
+    Feedbag.find(uri)
+  end
   
 end

data/app/models/feed_parent.rb

@@ -0,0 +1,18 @@
+# == Schema Information
+#
+# Table name: feed_parents
+#
+#  id           :integer(4)      not null, primary key
+#  feed_id      :integer(4)
+#  ownable_id   :integer(4)
+#  ownable_type :string(255)
+#  created_at   :datetime
+#  updated_at   :datetime
+#
+
+class FeedParent < ActiveRecord::Base
+  unloadable
+  
+  belongs_to :feed
+  belongs_to :ownable, :polymorphic => true
+end

data/app/models/language.rb

@@ -1,2 +1,16 @@
+# == Schema Information
+#
+# Table name: languages
+#
+#  id                   :integer(4)      not null, primary key
+#  name                 :string(255)
+#  english_name         :string(255)
+#  locale               :string(255)
+#  supported            :boolean(1)      default(TRUE)
+#  is_default           :integer(4)      default(0)
+#  muck_raker_supported :boolean(1)
+#  indexed_records      :integer(4)      default(0)
+#
+
 class Language < ActiveRecord::Base
 end

data/app/models/oai_endpoint.rb

@@ -1,2 +1,14 @@
+# == Schema Information
+#
+# Table name: oai_endpoints
+#
+#  id              :integer(4)      not null, primary key
+#  uri             :string(2083)
+#  display_uri     :string(2083)
+#  metadata_prefix :string(255)
+#  title           :string(1000)
+#  short_title     :string(100)
+#
+
 class OaiEndpoint < ActiveRecord::Base
 end

data/app/models/raker_mailer.rb

@@ -0,0 +1,16 @@
+class RakerMailer < ActionMailer::Base
+  unloadable
+  
+  layout 'email_default'
+  default_url_options[:host] = GlobalConfig.application_url
+ 
+  def notification_feed_added(feed)
+    recipients  GlobalConfig.admin_email
+    from        "#{GlobalConfig.from_email_name} <#{GlobalConfig.from_email}>"
+    sent_on     Time.now
+    subject     I18n.t('muck.raker.new_global_feed', :application_name => GlobalConfig.application_name)
+    body        :feed => feed,
+                :application_name => GlobalConfig.application_name
+  end
+
+end

data/app/models/recommendation.rb

@@ -1,2 +1,15 @@
+# == Schema Information
+#
+# Table name: recommendations
+#
+#  id               :integer(4)      not null, primary key
+#  entry_id         :integer(4)
+#  dest_entry_id    :integer(4)
+#  rank             :integer(4)
+#  relevance        :decimal(8, 6)   default(0.0)
+#  clicks           :integer(4)      default(0)
+#  avg_time_at_dest :integer(4)      default(60)
+#
+
 class Recommendation < ActiveRecord::Base
 end

data/app/models/service.rb

@@ -1,2 +1,16 @@
+# == Schema Information
+#
+# Table name: services
+#
+#  id                :integer(4)      not null, primary key
+#  uri               :string(2083)    default("")
+#  name              :string(1000)    default("")
+#  api_uri           :string(2083)    default("")
+#  uri_template      :string(2083)    default("")
+#  icon              :string(2083)    default("rss.gif")
+#  sequence          :integer(4)
+#  requires_password :boolean(1)
+#
+
 class Service < ActiveRecord::Base
 end

data/app/models/subject.rb

@@ -1,2 +1,10 @@
+# == Schema Information
+#
+# Table name: subjects
+#
+#  id   :integer(4)      not null, primary key
+#  name :string(255)
+#
+
 class Subject < ActiveRecord::Base
 end