muck-raker 0.1.0

data/db/bootstrap/oai_endpoints.yml

@@ -0,0 +1,6 @@
+oer_commons:
+  uri: 'http://www.oercommons.org/oai'
+  display_uri: 'http://www.oercommons.org'
+  metadata_prefix: 'oer_recommender'
+  title: 'OER Commons - Open Educational Resources'
+  short_title: 'OERC'

data/db/bootstrap/services.yml

@@ -0,0 +1,172 @@
+rss:
+    id: 1
+    title: rss
+    uri:
+    api_uri:
+    uri_template:
+    icon: rss-atom.gif
+    sequence: 1
+oai:
+    id: 2
+    title: OAI 
+    uri: 
+    api_uri: 
+    uri_template: 
+    icon: oai.gif 
+    sequence: 2
+twitter:
+    id: 3 
+    title: Twitter
+    uri: "http://www.twitter.com"
+    api_uri: 
+    uri_template: 
+    icon: twitter.gif
+    sequence: 3
+youtube:
+    id: 4 
+    title: You Tube 
+    uri: "http://www.youtube.com"
+    api_uri: 
+    uri_template: feed://www.youtube.com/rss/user/{username}/videos.rss
+    icon: youtube.gif
+    sequence: 4
+flickr:
+    id: 5 
+    title: Flickr 
+    uri: "http://www.flickr.com"
+    api_uri: 
+    uri_template: "http://api.flickr.com/services/feeds/activity.gne?id={username}"
+    icon: flickr.gif
+    sequence: 5
+delicious:
+    id: 6
+    title: del.icio.us 
+    uri: "http://del.icio.us/"
+    api_uri: "https://api.del.icio.us/v1/"
+    uri_template: 
+    icon: delicious.gif
+    sequence: 6
+digg:
+    id: 7 
+    title: Digg 
+    uri: "http://www.digg.com"
+    api_uri: 
+    uri_template: "http://digg.com/rss/{username}/index2.xml"
+    icon: digg.gif
+    sequence: 7
+43places:
+    id: 8 
+    title: 43 Places
+    uri: "http://www.43places.com"
+    api_uri: 
+    uri_template: "http://www.43places.com/rss/uber/author?username={username}"
+    icon: 43places.gif
+    sequence: 8
+allconsuming:
+    id: 9 
+    title: All Consuming 
+    uri: "http://allconsuming.net/"
+    api_uri: 
+    uri_template: "http://allconsuming.net/person/{username}/rss"
+    icon: allconsuming.gif
+    sequence: 9
+clipmarks:
+    id: 10 
+    sequence: 10
+    title: Clipmarks
+    uri: "http://www.clipmarks.com"
+    api_uri: 
+    uri_template: "http://rss.clipmarks.com/clipper/{username}/"
+    icon: clipmarks.gif
+lastfm:
+    id: 11 
+    sequence: 11
+    title: Last.fm
+    uri: "http://www.last.fm"
+    api_uri: 
+    uri_template: "http://ws.audioscrobbler.com/1.0/user/{username}/recenttracks.rss"
+    icon: lastfm.gif
+livejournal:
+    id: 12 
+    sequence: 12
+    title: LiveJournal
+    uri: "http://www.livejournal.com"
+    api_uri: 
+    uri_template: "http://{username}.livejournal.com"
+    icon: livejournal.gif
+simpy:
+    id: 13 
+    sequence: 13
+    title: Simpy
+    uri: "http://www.simpy.com"
+    api_uri: 
+    uri_template: "http://www.simpy.com/rss/user/{username}/links/"
+    icon: simpy.gif
+wordpress:
+    id: 14 
+    sequence: 14
+    title: Wordpress.com
+    uri: "http://www.wordpress.com/"
+    api_uri: 
+    uri_template: "http://{username}.wordpress.com/"
+    icon: wordpress.gif
+xanga:
+    id: 15 
+    sequence: 15
+    title: Xanga
+    uri: "http://www.xanga.com"
+    api_uri: 
+    uri_template: "http://www.xanga.com/{username}/rss"
+    icon: xanga.gif
+citeulike:
+    id: 16 
+    sequence: 16
+    title: cite u like
+    uri: "http://www.citeulike.org"
+    api_uri: 
+    uri_template: "http://www.citeulike.org/rss/user/{username}"
+    icon: citeulike.gif
+googlenews:
+    id: 17 
+    sequence: 17
+    title: Google News
+    uri: "http://news.google.com/"
+    icon: googlenews.gif
+googleblogsearch:
+    id: 18 
+    sequence: 18
+    title: Google Blog Search
+    uri: "http://blogsearch.google.com/"
+    icon: googleblogs.gif
+opml:
+    id: 19 
+    sequence: 19
+    title: OPML
+    icon: opml.gif
+tag:
+    id: 20 
+    sequence: 20
+    title: Tag
+    icon: tag.gif
+odeo:
+    id: 21 
+    sequence: 21
+    title: Odeo Podcasts
+    uri: "http://www.odeo.com"
+    icon: odeo.gif
+technorati:
+    id: 22 
+    sequence: 22
+    title: Technorati
+    uri: "http://www.technorati.com/"
+    api_uri: 
+    uri_template: "http://feeds.technorati.com/faves/{username}?format=rss"
+    icon: technorati.gif
+merlot:
+    id: 23 
+    sequence: 23
+    title: Merlot
+    uri: "http://www.merlot.org/"
+    api_uri: 
+    uri_template: 
+    icon: merlot.gif

data/db/migrate/20090602191243_create_muck_raker.rb

@@ -0,0 +1,304 @@
+class CreateMuckRaker < ActiveRecord::Migration
+
+  def self.up
+
+    create_table "action_types", :force => true do |t|
+      t.string  "action_type"
+      t.integer "weight"
+    end
+
+    create_table "aggregation_feeds", :force => true do |t|
+      t.integer "aggregation_id"
+      t.integer "feed_id"
+    end
+
+    add_index "aggregation_feeds", ["aggregation_id"]
+    add_index "aggregation_feeds", ["feed_id"]
+
+    create_table "aggregations", :force => true do |t|
+      t.string   "name"
+      t.string   "title"
+      t.text     "description"
+      t.text     "top_tags"
+      t.integer  "user_id"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+    end
+
+    add_index "aggregations", ["user_id"]
+
+    create_table "attentions", :force => true do |t|
+      t.integer "attentionable_id"
+      t.string  "attentionable_type"
+      t.integer "entry_id"
+      t.string  "action_type"
+      t.float   "weight"
+    end
+
+    create_table "clicks", :force => true do |t|
+      t.integer  "recommendation_id"
+      t.datetime "when",                              :null => false
+      t.string   "referrer",          :limit => 2083
+      t.string   "requester"
+      t.string   "user_agent",        :limit => 2083
+    end
+
+    add_index "clicks", ["recommendation_id"]
+    add_index "clicks", ["referrer"]
+    add_index "clicks", ["requester"]
+    add_index "clicks", ["user_agent"]
+    add_index "clicks", ["when"]
+
+    create_table "entries", :force => true do |t|
+      t.integer  "feed_id",                                                                    :null => false
+      t.string   "permalink",               :limit => 2083, :default => "",                    :null => false
+      t.string   "author",                  :limit => 2083
+      t.text     "title",                                                                      :null => false
+      t.text     "description"
+      t.text     "content"
+      t.boolean  "unique_content",                          :default => false
+      t.datetime "published_at",                                                               :null => false
+      t.datetime "entry_updated_at"
+      t.datetime "harvested_at"
+      t.string   "oai_identifier",          :limit => 2083
+      t.integer  "language_id"
+      t.string   "direct_link",             :limit => 2083
+      t.datetime "indexed_at",                              :default => '1971-01-01 01:01:01', :null => false
+      t.datetime "relevance_calculated_at",                 :default => '1971-01-01 01:01:01', :null => false
+      t.text     "popular"
+      t.text     "relevant"
+      t.text     "other"
+    end
+
+    add_index "entries", ["direct_link"]
+    add_index "entries", ["feed_id"]
+    add_index "entries", ["indexed_at"]
+    add_index "entries", ["language_id"]
+    add_index "entries", ["oai_identifier"]
+    add_index "entries", ["permalink"]
+    add_index "entries", ["published_at"]
+    add_index "entries", ["relevance_calculated_at"]
+
+    create_table "entries_users", :force => true do |t|
+      t.integer  "entry_id",                           :null => false
+      t.integer  "user_id",         :default => 0
+      t.boolean  "clicked_through", :default => false
+      t.datetime "created_at"
+    end
+
+    add_index "entries_users", ["entry_id", "user_id"]
+    add_index "entries_users", ["entry_id"]
+    add_index "entries_users", ["user_id"]
+
+    create_table "entry_images", :force => true do |t|
+      t.integer "entry_id"
+      t.string  "uri",      :limit => 2083
+      t.string  "link",     :limit => 2083
+      t.string  "alt"
+      t.string  "title"
+      t.integer "width"
+      t.integer "height"
+    end
+
+    add_index "entry_images", ["entry_id"]
+
+    create_table "feeds", :force => true do |t|
+      t.string   "uri",                        :limit => 2083
+      t.string   "display_uri",                :limit => 2083
+      t.string   "title",                      :limit => 1000
+      t.string   "short_title",                :limit => 100
+      t.text     "description"
+      t.string   "tag_filter",                 :limit => 1000
+      t.text     "top_tags"
+      t.integer  "priority",                                   :default => 10
+      t.integer  "status",                                     :default => 1
+      t.datetime "last_requested_at"
+      t.datetime "last_harvested_at"
+      t.integer  "harvest_interval",                           :default => 86400
+      t.integer  "failed_requests",                            :default => 0
+      t.text     "error_message"
+      t.integer  "service_id",                                 :default => 0,     :null => false
+      t.string   "login"
+      t.string   "password"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.datetime "entries_changed_at"
+      t.string   "harvested_from_display_uri", :limit => 2083
+      t.string   "harvested_from_title",       :limit => 1000
+      t.string   "harvested_from_short_title", :limit => 100
+      t.integer  "entries_count"
+      t.integer  "default_language_id", :default => 0
+      t.boolean  "ocw", :default => false
+    end
+
+    add_index "feeds", ["service_id"]
+    add_index "feeds", ["uri"]
+
+    create_table "micro_concerts", :force => true do |t|
+      t.integer "micro_event_id"
+      t.string  "performer"
+      t.string  "ticket_url",     :limit => 2083
+    end
+
+    add_index "micro_concerts", ["micro_event_id"]
+
+    create_table "micro_conferences", :force => true do |t|
+      t.integer  "micro_event_id"
+      t.string   "theme"
+      t.datetime "register_by"
+      t.datetime "submit_by"
+    end
+
+    add_index "micro_conferences", ["micro_event_id"]
+
+    create_table "micro_event_links", :force => true do |t|
+      t.integer "micro_event_id"
+      t.string  "uri"
+      t.string  "title"
+    end
+
+    add_index "micro_event_links", ["micro_event_id"]
+
+    create_table "micro_event_people", :force => true do |t|
+      t.integer "micro_event_id"
+      t.string  "name"
+      t.string  "role"
+      t.string  "email"
+      t.string  "link",           :limit => 2083
+      t.string  "phone"
+    end
+
+    add_index "micro_event_people", ["micro_event_id"]
+
+    create_table "micro_events", :force => true do |t|
+      t.integer  "entry_id",    :null => false
+      t.string   "name",        :null => false
+      t.text     "description"
+      t.string   "price"
+      t.text     "image"
+      t.text     "address"
+      t.text     "subaddress"
+      t.string   "city"
+      t.string   "state"
+      t.string   "postcode"
+      t.string   "country"
+      t.datetime "begins",      :null => false
+      t.datetime "ends"
+      t.text     "tags"
+      t.string   "duration"
+      t.text     "location"
+    end
+
+    add_index "micro_events", ["entry_id"]
+
+    create_table "oai_endpoints", :force => true do |t|
+      t.string "uri",             :limit => 2083
+      t.string "display_uri",     :limit => 2083
+      t.string "metadata_prefix"
+      t.string "title",           :limit => 1000
+      t.string "short_title",     :limit => 100
+    end
+
+    create_table "personal_recommendations", :force => true do |t|
+      t.integer "personal_recommendable_id"
+      t.string  "personal_recommendable_type"
+      t.integer "destination_id"
+      t.string  "destination_type"
+      t.integer "rank"
+      t.float   "relevance"
+    end
+
+    create_table "queries", :force => true do |t|
+      t.text    "name"
+      t.integer "frequency"
+    end
+
+    create_table "recommendations", :force => true do |t|
+      t.integer "entry_id"
+      t.integer "dest_entry_id"
+      t.integer "rank"
+      t.decimal "relevance",        :precision => 8, :scale => 6, :default => 0.0
+      t.integer "clicks",                                         :default => 0
+      t.integer "avg_time_at_dest",                               :default => 60
+    end
+
+    add_index "recommendations", ["dest_entry_id"]
+    add_index "recommendations", ["entry_id"]
+
+    create_table "services", :force => true do |t|
+      t.string  "uri",               :limit => 2083, :default => ""
+      t.string  "title",             :limit => 1000, :default => ""
+      t.string  "api_uri",           :limit => 2083, :default => ""
+      t.string  "uri_template",      :limit => 2083, :default => ""
+      t.string  "icon",              :limit => 2083, :default => 'rss.gif'
+      t.integer "sequence"
+      t.boolean "requires_password",                 :default => false
+    end
+
+    create_table "watched_pages", :force => true do |t|
+      t.integer  "entry_id"
+      t.datetime "harvested_at"
+      t.boolean  "has_microformats", :default => false
+      t.datetime "created_at"
+      t.datetime "updated_at"
+    end
+
+    add_index "watched_pages", ["entry_id"]
+
+    create_table "subjects", :force => true do |t|
+      t.string   "name"
+    end
+    
+    create_table "entries_subjects", :force => true do |t| 
+      t.integer  "subject_id"
+      t.integer  "entry_id"
+      t.boolean  "autogenerated", :default => false
+    end
+    execute "ALTER TABLE entries_subjects DROP COLUMN id, DROP PRIMARY KEY, ADD PRIMARY KEY USING BTREE(subject_id, entry_id);"
+    add_index "entries_subjects", ["subject_id"]
+    add_index "entries_subjects", ["entry_id"]
+    add_index "entries_subjects", ["autogenerated"]
+
+    create_table "cloud_caches", :force => true do |t| 
+      t.integer  "language_id"
+      t.string   "filter"
+      t.string   "tag_list", :limit => 2500
+    end
+    execute "ALTER TABLE cloud_caches DROP COLUMN id, DROP PRIMARY KEY, ADD PRIMARY KEY USING BTREE(language_id, filter);"
+
+    # add a flag to the languages table to flag languages we support
+    add_column :languages, :muck_raker_supported, :boolean, :default => false
+    add_column :languages, :indexed_records, :integer, :default => 0
+    add_index "languages", ["muck_raker_supported"]
+  end
+
+  def self.down
+    drop_table :action_types
+    drop_table :aggregation_feeds
+    drop_table :aggregation_top_tags
+    drop_table :aggregations
+    drop_table :attentions
+    drop_table :clicks
+    drop_table :entries
+    drop_table :subjects
+    drop_table :entries_subjects
+    drop_table :cloud_caches
+    drop_table :entries_users
+    drop_table :entry_images
+    drop_table :feeds
+    drop_table :interests
+    drop_table :recommender_languages
+    drop_table :micro_concerts
+    drop_table :micro_conferences
+    drop_table :micro_event_links   
+    drop_table :micro_event_people
+    drop_table :micro_events
+    drop_table :oai_endpoints
+    drop_table :personal_recommendations
+    drop_table :queries
+    drop_table :recommendations
+    drop_table :services
+    drop_table :watched_pages
+  end
+
+end

data/install.rb

@@ -0,0 +1 @@
+# Install hook code here

data/lib/muck_raker/initialize_routes.rb

@@ -0,0 +1,8 @@
+class ActionController::Routing::RouteSet
+  def load_routes_with_muck_raker!
+    muck_raker_routes = File.join(File.dirname(__FILE__), *%w[.. .. config muck_raker_routes.rb])
+    add_configuration_file(muck_raker_routes) unless configuration_files.include? muck_raker_routes
+    load_routes_without_muck_raker!
+  end
+  alias_method_chain :load_routes!, :muck_raker
+end

data/lib/muck_raker/languages.rb

@@ -0,0 +1,53 @@
+module Recommender
+  module Languages
+    AVAILABLE_LOCALES = "^en|es|fr|nl|ja|de|ru|zh"
+    DEFAULT_LOCALE = 'en'
+    
+    ARABIC = "ar"
+    BULGARIAN = "bg"
+    CATALAN = "ca"
+    CHINESE = "zh"
+    CHINESE_SIMPLIFIED = "zh-CN"
+    CHINESE_TRADITIONAL = "zh-TW"
+    CROATIAN = "cr"
+    CZECH = "cs"
+    DANISH = "da"
+    DUTCH = "nl"
+    ENGLISH = "en"
+    FILIPINO = "tl"
+    FINNISH = "fi"
+    FRENCH = "fr"
+    GERMAN = "de"
+    GREEK = "el"
+    HEBREW = "iw"
+    HINDI = "hi"
+    INDONESIAN = "id"
+    ITALIAN = "it"
+    JAPANESE = "ja"
+    KOREAN = "ko"
+    LATVIAN = "lv"
+    LITHUANIAN = "lt"
+    NORWEGIAN = "no"
+    POLISH = "pl"
+    PORTUGESE = "pt"
+    ROMANIAN = "ro"
+    RUSSIAN = "ru"
+    SERBIAN = "sr"
+    SLOVAK = "sk"
+    SLOVENIAN = "sl"
+    SPANISH = "es"
+    SWEDISH = "sv"
+    UKRANIAN = "uk"
+    VIETNAMESE = "vi"
+
+    # LANGUAGES = [ARABIC, BULGARIAN, CATALAN, CHINESE, CHINESE_SIMPLIFIED,CHINESE_TRADITIONAL,
+    #  CROATIAN, CZECH, DANISH, DUTCH, ENGLISH, FILIPINO, FRENCH, GERMAN, GREEK, HEBREW,
+    #  ITALIAN, JAPANESE, KOREAN, LATVIAN, LITHUANIAN, NORWEGIAN, POLISH, PORTUGESE,
+    #  ROMANIAN, RUSSIAN, SERBIAN, SLOVAK, SLOVENIAN, SPANISH, SWEDISH, UKRANIAN, VIETNAMESE]
+
+     LANGUAGES = [ARABIC, BULGARIAN, CATALAN, CHINESE, CHINESE_SIMPLIFIED,CHINESE_TRADITIONAL,
+                  CZECH, DANISH, DUTCH, ENGLISH, FILIPINO, FRENCH, GERMAN, GREEK, HEBREW,
+                  ITALIAN, JAPANESE, KOREAN, LATVIAN, LITHUANIAN, NORWEGIAN, POLISH, PORTUGESE,
+                  ROMANIAN, RUSSIAN, SERBIAN, SLOVAK, SLOVENIAN, SPANISH, SWEDISH, UKRANIAN, VIETNAMESE]
+  end
+end

data/lib/muck_raker.rb

@@ -0,0 +1,6 @@
+
+module MuckRaker
+end
+
+Mime::Type.register "application/rdf+xml", :rdf
+Mime::Type.register "text/javascript", :pjs

data/locales/en.yml

@@ -0,0 +1,42 @@
+en:
+  muck:
+    raker:
+        no_tags: No tags available yet
+        title: Folksemantic
+        collections: Collections
+        demo: Demo
+        tag_line: making open education work
+        developers: Developers
+        search_button: Go
+        collections_long: Indexed Collections
+        developers_long: Information for Developers
+        powered_by: powered by
+        no_hits: No hits. Please try a different search or choose a different language.
+        previous: Previous
+        next: Next
+        results: Results
+        of: of
+        for: for
+        relevance: relevance
+        metadata: metadata
+        related_resources: related
+        rss: RSS 
+        xml: XML 
+        atom: Atom 
+        rdf: RDF 
+        related_resources_title: Related Resources
+        calculated: calculated
+        details: Details
+        gm_title: Related Resources
+        gm_more_prompt: More Related Resources
+        direct_link: '[direct link]'
+        register_link: Sign Up!
+        register_description: 'Sign up to receive personalized <span class="highlight">recommendations</span>, participate in <span class="highlight">groups</span>, and easily <span class="highlight">publish to blogs</span>.'
+        widgets_link: Get Widgets!
+        widgets_description: Get widgets to help users <span class="highlight">find</span> resources and <span class="highlight">discover resources related</span> to ones on your website.
+        tour: Take a Tour!
+        tour_description: Watch a <span class="highlight">video introduction</span> of Folksemantic.
+        search: search
+        search_link: Find Resources!
+        search_description: Find learning resources by <span class="highlight">searching</span> and <span class="highlight">browsing</span> over <span class="highlight">110,000</span> open educational resources.
+        welcome_user: "Welcome {{user}}"

data/rails/init.rb

@@ -0,0 +1,27 @@
+ActiveSupport::Dependencies.load_once_paths << lib_path
+
+if config.respond_to?(:gems)
+  config.gem "mbleigh-acts-as-taggable-on", :source => "http://gems.github.com", :lib => "acts-as-taggable-on"
+else
+  begin
+    require 'acts-as-taggable-on'
+  rescue LoadError
+    begin
+      gem 'mbleigh-acts-as-taggable-on'
+    rescue Gem::LoadError
+      puts "Please install the acts-as-taggable-on gem from http://gems.github.com"
+    end
+  end
+end
+
+I18n.load_path += Dir[ File.join(RAILS_ROOT, 'vendor', 'plugins', 'muck_raker', 'locales', '*.{rb,yml}') ]
+
+ENV['APP_AVAILABLE_LOCALES'] = "^en|es|fr|nl|ja|de|ru|zh"
+ENV['APP_DEFAULT_LOCALE'] = 'en'
+
+require 'muck_raker'
+require 'muck_raker/initialize_routes'
+
+config.to_prepare do
+  ApplicationController.helper(MuckRakerHelper)
+end

data/raker/lib/aggregatord.properties

@@ -0,0 +1,45 @@
+# number of worker threads to spawn
+#threads=20
+
+# size of the queue
+#stale_feed_queue_size=100
+
+# seconds to wait before timing out a HTTP request
+request_timeout=90
+
+# seconds to wait before querying the database for stale feeds
+#stale_thread_poll_interval=5
+
+# minutes to wait before checking to see if a feed has new entries
+#feed_refresh_interval=60
+
+# whether or not to log to file
+#log_file_prefix=recommender
+
+# discover new feeds provided by registered oai endpoints
+discover_oai_sets=true 
+
+# import archived feed data from disk before requesting it across the wire
+import_archived_feed_data=false
+
+# request feeds across the wire from their servers
+harvest_from_wire=true
+
+# whether or not to log to console
+log_to_console=true
+
+# rails environment (default is production)
+rails_env=production
+
+# path to database.yml (default is ../../../../../shared/system/config/database.yml)
+db_yml=../../../../../shared/system/config/database.yml
+
+# amount of info to log
+# NEVER		= 0;
+# CRITICAL	= 1;
+# EXCEPTION	= 2;
+# WARNING	= 3;
+# STATUS	= 4;
+# INFO		= 5;
+# ALL		= 10;
+debug_level=10