murlsh 1.4.1 → 1.5.0

data/.gitignore

@@ -0,0 +1,4 @@
+*~
+\#*#
+\.#*
+*\.gem

data/.htaccess

@@ -1,11 +1,8 @@
 Options -Indexes
 
-AddOutputFilterByType DEFLATE application/atom+xml
 AddOutputFilterByType DEFLATE application/javascript
-AddOutputFilterByType DEFLATE application/rss+xml
 AddOutputFilterByType DEFLATE application/xhtml+xml
 AddOutputFilterByType DEFLATE application/xml
-AddOutputFilterByType DEFLATE audio/x-mpegurl
 AddOutputFilterByType DEFLATE text/css
 AddOutputFilterByType DEFLATE text/html
 

data/Gemfile

@@ -0,0 +1,41 @@
+source :rubygems
+
+# This does not use 'gemspec' because it creates a Gemfile.lock that treats
+# murlsh as a "gem" and will not work on Heroku, which sees murlsh as an
+# "app".
+
+# Dependencies duplicated in gemspec and here until a better solution is
+# found.
+
+%w{
+  activerecord >= 2.3.4
+  aws-s3 ~> 0.6
+  bcrypt-ruby >= 2.1.2
+  builder > 0
+  htmlentities >= 4.2.0
+  json >= 1.2.3
+  nokogiri ~> 1.0
+  plumnailer >= 0.1.3
+  postrank-uri ~> 1.0
+  public_suffix_service ~> 0.0
+  push-notify >= 0.1.0
+  rack >= 1.0.0
+  rack-cache >= 0.5.2
+  rack-rewrite >= 1.0.2
+  rack-throttle >= 0.3.0
+  rmagick >= 1.15.14
+  rmail ~> 1.0
+  sqlite3 ~> 1.3
+  tinyatom >= 0.3.4
+  treetop ~> 1.4
+  twitter >= 0.9.12
+  }.each_slice(3) { |g,o,v| gem g, "#{o} #{v}" }
+
+group :development do
+  %w{
+    fakeweb ~> 1.3
+    flog >= 2.5.0
+    rack-test ~> 0.5
+    rspec ~> 2.0
+    }.each_slice(3) { |g,o,v| gem g, "#{o} #{v}" }
+end

data/README.textile

@@ -6,7 +6,7 @@ Host your bookmarks or maintain a link blog.
 * generates Atom and RSS feeds
 * generates podcast RSS feed and m3u file for all audio urls
 * generates json and jsonp feeds for client-side inclusion in other sites
-* search
+* search, all output formats can be filtered by search criteria
 * uses HTML5 audio for mp3 and ogg urls
 * looks good on iPhone
 * PubSubHubbub notification
@@ -14,6 +14,7 @@ Host your bookmarks or maintain a link blog.
 * rack interface
 * Gravatar support
 * generates import scripts from delicious api exports
+* optionally store thumbnails in S3
 
 See "http://urls.matthewm.boedicker.org/":http://urls.matthewm.boedicker.org/ for example.
 
@@ -34,13 +35,60 @@ rake init
 </code>
 </pre>
 
+h2. Heroku
+
+<pre>
+<code>
+gem install heroku
+heroku keys:add
+</code>
+</pre>
+
+Create a fork on github and clone it or clone public url:
+
+<pre>
+<code>
+git clone git://github.com/mmb/murlsh.git
+cd murlsh
+bundle install
+heroku create <choose a name>
+rake heroku:config
+heroku info
+rake config[root_url,<your app's Heroku url>]
+rake config[s3_bucket,<your S3 bucket name>]
+rake config[s3_id,<your S3 id>]
+rake config[s3_secret,<your S3 secret>]
+</code>
+</pre>
+
+S3 is used for thumbnail storage because Heroku cannot write local files.
+
+Other config in config.yaml is optional.
+
+<pre>
+<code>
+rake user:add
+git add .
+git commit
+git push heroku master
+heroku rake db:migrate
+</code>
+</pre>
+
 h2. Development
 
-* Create a fork and check it out
-* edit config.yaml
-* rake init
-* rackup
-* open http://localhost:9292/
+Create a fork on Github and clone it.
+
+<pre>
+<code>
+rake config[root_url,http://localhost:9292/]
+rake db:migrate
+rake user:add
+rackup
+</code>
+</pre>
+
+Browse to http://localhost:9292/
 
 h1. Updating
 
@@ -60,6 +108,18 @@ h2. Recent urls
 * http://your_root/json.json
 * http://your_root/json.json?callback=x (jsonp)
 
+h1. Thumbnails
+
+Thumbnail images are generated from added urls using plumnailer. They are
+scaled down to 'thumbnail_max_side' in config.yaml and stored locally.
+
+Thumbnails can also be manually specified by passing their url as the
+'thumbnail' parameter when adding a url. They are also scaled and stored
+locally.
+
+The plugin add_pre_40_thumbnail_shortcuts.rb can be used to specify
+short names for frequently used thumbnail urls that can be passed in instead.
+
 h1. Plugins
 
 Classes in the plugins directory can be used to change behavior at certain
@@ -79,7 +139,9 @@ Plugin hooks
 |add_pre|called before a new url is saved|url, config hash|undefined|
 |add_post|called after a new url is saved|url, config hash|undefined|
 |avatar|called to get an avatar url from an email md5 sum|avatar url, url, config hash|avatar url|
+|store_asset|store an asset somewhere where it can be loaded by url|name, data, config hash|asset url if successfully stored|
 |url_display_add|called to display additional information after urls|markup builder, url, config hash|undefined|
+|url_display_pre|called to modify a url on-the-fly before display, does not change database|url, rack request, config hash|undefined|
 
 h1. PubSubHubbub
 

data/Rakefile

@@ -1,8 +1,8 @@
 $:.unshift(File.join(File.dirname(__FILE__), 'lib'))
 
-require 'cgi'
 require 'digest/md5'
 require 'net/http'
+require 'logger'
 require 'open-uri'
 require 'pp'
 require 'set'
@@ -11,18 +11,23 @@ require 'yaml'
 
 require 'active_record'
 require 'RMagick'
-require 'sqlite3'
 
 require 'murlsh'
 
 def gem_not_found(gem_name)
-  puts "#{gem_name} not found, install it with: gem install #{gem_name}"
+  puts "#{gem_name} gem not found"
 end
 
 config = YAML.load_file('config.yaml')
 
+# for Heroku
+db_config_file = File.join(File.dirname(__FILE__), 'config', 'database.yml')
+if File.exist?(db_config_file)
+  config['db'] = YAML.load_file(db_config_file)['production']
+end
+
 desc 'Initialize a new installation.'
-task :init => %w{db:init user:add compress} do
+task :init => %w{db:migrate user:add compress} do
   puts <<-eos
 
 Things you might want to do now:
@@ -46,8 +51,7 @@ namespace :db do
 
   desc 'Delete the last url added.'
   task :delete_last_url do
-    ActiveRecord::Base.establish_connection(:adapter => 'sqlite3',
-      :database => config.fetch('db_file'))
+    ActiveRecord::Base.establish_connection config.fetch('db')
 
     last = Murlsh::Url.find(:last, :order => 'time')
     pp last
@@ -57,46 +61,43 @@ namespace :db do
 
   desc 'Check for duplicate URLs.'
   task :dupcheck do
-    db = SQLite3::Database.new(config.fetch('db_file'))
-    db.results_as_hash = true
+    ActiveRecord::Base.establish_connection config.fetch('db')
+
     h = {}
-    db.execute('SELECT * FROM urls').each do |r|
-      h[r['url']] = h.fetch(r['url'], []).push([r['id'], r['time']])
+    Murlsh::Url.all.each do |mu|
+      h[mu.url] = h.fetch(mu.url, []).push([mu.id, mu.time])
     end
     h.find_all { |k,v| v.size > 1 }.each do |k,v|
       puts k
-      v.each { |id,time| puts "  #{id} #{time}" }
+      v.each { |id,time| puts "  id #{id} (#{time})" }
     end
   end
 
-  desc 'Create an empty database.'
-  task :init do
-    puts "creating #{config.fetch('db_file')}"
-    db = SQLite3::Database.new(config.fetch('db_file'))
-    db.execute 'CREATE TABLE urls (
-      id INTEGER PRIMARY KEY,
-      time TIMESTAMP,
-      url TEXT,
-      email TEXT,
-      name TEXT,
-      title TEXT,
-      content_length INTEGER,
-      content_type TEXT,
-      via TEXT,
-      thumbnail_url TEXT);
-      '
-    db.execute 'CREATE INDEX IF NOT EXISTS urls_time_desc ON urls (time DESC);'
+  desc 'Migrate the database.'
+  task :migrate do
+    ActiveRecord::Base.establish_connection config.fetch('db')
+    ActiveRecord::Base.logger = Logger.new($stdout)
+    ActiveRecord::Migration.verbose = true
+    ActiveRecord::Migrator.migrate 'db/migrate'
   end
 
   desc 'Interact with the database.'
   task :shell do
-    exec "sqlite3 #{config['db_file']}"
+    db = config.fetch('db')
+    command = case db.fetch('adapter')
+      when 'sqlite3'; "sqlite3 #{db.fetch('database')}"
+    end
+
+    if command
+      exec command
+    else
+      puts "Don't know how to launch shell for database '#{db.fetch('adapter')}'"
+    end
   end
 
   desc 'Search urls and titles in the database.'
   task :grep, :search do |t,args|
-    ActiveRecord::Base.establish_connection(:adapter => 'sqlite3',
-      :database => config.fetch('db_file'))
+    ActiveRecord::Base.establish_connection config.fetch('db')
 
     like = "%#{args.search}%"
     Murlsh::Url.all(:conditions =>
@@ -140,7 +141,7 @@ begin
     t.verbose = true
   end
 rescue LoadError
-  puts 'rspec gem ~> 2.0 not found, to enable test task: gem install rspec'
+  gem_not_found 'rspec'
 end
 
 desc 'Test remote title fetch for a URL and show errors.'
@@ -151,8 +152,7 @@ end
 
 desc 'Try to fetch the title for a url and update it in the database.'
 task :title_fetch, :url_id do |t, args|
-  ActiveRecord::Base.establish_connection(:adapter => 'sqlite3',
-    :database => config.fetch('db_file'))
+  ActiveRecord::Base.establish_connection config.fetch('db')
   url = Murlsh::Url.find(args.url_id)
   puts "Url: #{url.url}"
   puts "Previous title: #{url.title}"
@@ -176,51 +176,6 @@ namespace :user do
 
 end
 
-# Validate a document with the W3C validation service.
-def validate_html(check_url, options={})
-  opts = {
-    :validator_host => 'validator.w3.org',
-    :validator_port => 80,
-    :validator_path =>
-      "/check?uri=#{CGI::escape(check_url)}&charset=(detect+automatically)&doctype=Inline&group=0",
-  }.merge options
-
-  net_http = Net::HTTP.new(opts[:validator_host], opts[:validator_port])
-  # net_http.set_debug_output(STDOUT)
-
-  net_http.start do |http|
-    resp = http.request_head(opts[:validator_path])
-    result = {
-      :response => resp
-    }
-    if Net::HTTPSuccess === resp
-      result.merge!(
-        :status =>  resp['X-W3C-Validator-Status'],
-        :errors => resp['X-W3C-Validator-Errors'],
-        :warnings => resp['X-W3C-Validator-Warnings']
-      )
-    end
-    result
-  end
-
-end
-
-namespace :validate do
-
-  desc 'Validate HTML.'
-  task :html do
-    check_url = config['root_url']
-    print "validating #{check_url} : "
-    result = validate_html(check_url)
-    if Net::HTTPSuccess === result[:response]
-      puts "#{result[:status]} (#{result[:errors]} errors, #{result[:warnings]} warnings)"
-    else
-      puts result[:response]
-    end
-  end
-
-end
-
 desc 'Generate a shell script that will post a new url.'
 task :post_sh do
   puts <<EOS
@@ -238,25 +193,14 @@ curl \\
 EOS
 end
 
-# Concatenate some files and return the result as a string.
-def cat(in_files, sep=nil)
-  result = ''
-  in_files.each do |fname|
-    open(fname) do |h|
-      while (line = h.gets) do; result << line; end
-      result << sep  if sep
-    end
-  end
-  result
-end
-
 directory 'public/css'
 
 namespace :css do
 
   desc 'Combine and compress css.'
   task :compress => ['public/css'] do
-    combined = cat(config['css_files'].map { |x| "public/#{x}" }, "\n")
+    combined = Murlsh.cat_files(
+      config['css_files'].map { |x| "public/#{x}" }, "\n")
 
     md5sum = Digest::MD5.hexdigest(combined)
 
@@ -271,11 +215,7 @@ namespace :css do
 
     unless config['css_compressed'] == compressed_url
       config['css_compressed'] = compressed_url
-      config.extend(Murlsh::YamlOrderedHash)
-      config.each_value do |v|
-        v.extend(Murlsh::YamlOrderedHash)  if v.is_a?(Hash)
-      end
-      open('config.yaml', 'w') { |f| YAML.dump(config, f) }
+      Murlsh.write_ordered_hash config, 'config.yaml'
       puts "updated config with css_compressed = #{compressed_url}"
     end
   end
@@ -290,7 +230,8 @@ namespace :js do
 
   desc 'Combine and compress javascript.'
   task :compress => ['public/js'] do
-    combined = cat(config['js_files'].map { |x| "public/#{x}" } )
+    combined = Murlsh.cat_files(
+      config['js_files'].map { |x| "public/#{x}" } )
 
     compressed = Net::HTTP.post_form(
       URI.parse('http://closure-compiler.appspot.com/compile'), {
@@ -313,11 +254,7 @@ namespace :js do
 
     unless config['js_compressed'] == compressed_url
       config['js_compressed'] = compressed_url
-      config.extend(Murlsh::YamlOrderedHash)
-      config.each_value do |v|
-        v.extend(Murlsh::YamlOrderedHash)  if v.is_a?(Hash)
-      end
-      open('config.yaml', 'w') { |f| YAML.dump(config, f) }
+      Murlsh.write_ordered_hash config, 'config.yaml'
       puts "updated config with js_compressed = #{compressed_url}"
     end
   end
@@ -326,7 +263,7 @@ namespace :js do
   task :jslint do
     local_jslint = 'jslint_rhino.js'
     open(local_jslint, 'w') do |f|
-      f.write(cat(%w{
+      f.write(Murlsh.cat_files(%w{
         https://github.com/AndyStricker/JSLint/raw/rhinocmdline/fulljslint.js
         https://github.com/AndyStricker/JSLint/raw/rhinocmdline/rhino.js
       }))
@@ -352,11 +289,10 @@ namespace :thumb do
 
   desc 'Check that local thumbnails in database are consistent with filesystem.'
   task :check do
-    ActiveRecord::Base.establish_connection :adapter => 'sqlite3',
-      :database => config.fetch('db_file')
+    ActiveRecord::Base.establish_connection config.fetch('db')
     used_thumbnails = Set.new
     Murlsh::Url.all(
-      :conditions => "thumbnail_url like 'img/thumb/%'").each do |u|
+      :conditions => "thumbnail_url LIKE 'img/thumb/%'").each do |u|
       identity = "url #{u.id} (#{u.url})"
 
       path = File.join(%w{public}.concat(File.split(u.thumbnail_url)))
@@ -424,50 +360,27 @@ EOS
 
 end
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = 'murlsh'
-    gemspec.summary = 'Host your bookmarks or maintain a link blog'
-    gemspec.description = 'Host your bookmarks or maintain a link blog'
-    gemspec.email = 'matthewm@boedicker.org'
-    gemspec.homepage = 'http://github.com/mmb/murlsh'
-    gemspec.authors = ['Matthew M. Boedicker']
-    gemspec.executables = %w{murlsh}
-
-    # gemspec.signing_key = '/home/mmb/src/keys/gem-private_key.pem'
-    # gemspec.cert_chain = %w{/home/mmb/src/keys/gem-public_cert.pem}
-
-    %w{
-      activerecord >= 2.3.4
-      bcrypt-ruby >= 2.1.2
-      builder >= 2.1.2
-      htmlentities >= 4.2.0
-      json >= 1.2.3
-      nokogiri ~> 1.0
-      plumnailer >= 0.1.0
-      postrank-uri ~> 1.0
-      public_suffix_service ~> 0.0
-      push-notify >= 0.1.0
-      rack >= 1.0.0
-      rack-cache >= 0.5.2
-      rack-rewrite >= 1.0.2
-      rack-throttle >= 0.3.0
-      rmagick >= 1.15.14
-      sqlite3 ~> 1.3
-      tinyatom >= 0.3.3
-      treetop ~> 1.4
-      twitter >= 0.9.12
-      }.each_slice(3) { |g,o,v| gemspec.add_dependency(g, "#{o} #{v}") }
-    %w{
-      fakeweb ~> 1.3
-      flog >= 2.5.0
-      rack-test ~> 0.5
-      rspec ~> 2.0
-      }.each_slice(3) do |g,o,v|
-      gemspec.add_development_dependency(g, "#{o} #{v}")
-    end
+desc 'Set options in config.yaml.'
+task :config, :key, :value do |t, args|
+  orig_value = config[args.key]
+  if args.value != orig_value
+    config[args.key] = args.value
+    Murlsh.write_ordered_hash config, 'config.yaml'
+    puts "updated '#{args.key}' '#{orig_value}' => '#{args.value}'"
+  else
+    puts "'#{args.key}' is already set to '#{args.value}'"
   end
-rescue LoadError
-  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+
+namespace :heroku do
+
+  desc 'Set config options for deployment on Heroku.'
+  task :config => %w{compress} do
+    config.delete 'cache_entitystore'
+    config.delete 'cache_metastore'
+
+    Murlsh.write_ordered_hash config, 'config.yaml'
+    puts 'removed cache_entitystore and cache_metastore from config, Rack::Cache disabled'
+  end
+
 end

data/config.ru

@@ -3,6 +3,7 @@ $:.unshift(File.join(File.dirname(__FILE__), 'lib'))
 require 'uri'
 require 'yaml'
 
+require 'rack'
 require 'rack/cache'
 require 'rack/rewrite'
 require 'rack/throttle'
@@ -11,18 +12,29 @@ require 'murlsh'
 
 config = YAML.load_file('config.yaml')
 
+# for Heroku
+db_config_file = File.join(File.dirname(__FILE__), 'config', 'database.yml')
+if File.exist?(db_config_file)
+  config['db'] = YAML.load_file(db_config_file)['production']
+end
+
 # use Rack::ShowExceptions
 # no more than 1024 requests per day per ip
 use Rack::Throttle::Daily, :max => 1024
-if config.key?('cache_metastore') and config.key?('cache_entitystore')
+
+if !config['cache_metastore'].to_s.empty? and
+  !config['cache_entitystore'].to_s.empty?
   use Rack::Cache,
     :verbose => true,
     :metastore => config['cache_metastore'],
     :entitystore => config['cache_entitystore']
 end
+
 use Rack::ConditionalGet
 use Murlsh::EtagAddEncoding
 use Rack::Deflater
+use Rack::Head
+use Rack::ETag
 use Murlsh::FarFutureExpires, :patterns => [
   %r{[\da-z]{32}\.(?:gif|jpe?g|png)$}i,
   %r{\.gen\.(css|js)$}
@@ -32,7 +44,6 @@ feed_url = URI.join(config.fetch('root_url'), config.fetch('feed_file'))
 use Murlsh::MustRevalidate, :patterns => %r{^#{Regexp.escape(feed_url.path)}$}
 
 use Rack::Static, :urls => %w{/css/ /img/ /js/}, :root => 'public'
-use Rack::Static, :urls => %w{/atom.atom /m3u.m3u /podcast.rss /rss.rss}
 
 use Rack::Rewrite do
   r301 '/atom.xml', feed_url.to_s

data/config.yaml

@@ -5,7 +5,9 @@ cache_metastore: file:tmp/cache/rack/meta
 css_files: 
 - css/jquery.jgrowl.css
 - css/screen.css
-db_file: murlsh.db
+db: 
+  adapter: sqlite3
+  database: murlsh.db
 feed_file: atom.atom
 gravatar_size: 32
 js_files: 
@@ -13,12 +15,12 @@ js_files:
 - js/jquery.jgrowl_compressed.js
 - js/twitter-text-1.3.1.js
 - js/js.js
-meta_tag_description: URLs found interesting by Matthew M. Boedicker
+meta_tag_description: 
 meta_tag_verify-v1: 
 meta_tag_viewport: width=device-width,minimum-scale=1.0,maximum-scale=1.0
 num_posts_feed: 25
 num_posts_page: 25
-page_title: mmb url share
+page_title: murlsh
 pubsubhubbub_hubs: []
 
 quick_search: 
@@ -29,6 +31,9 @@ quick_search:
   video: hulu.com vimeo.com www.ted.com youtube.com
   wikipedia: wikipedia.org
 root_url: http://urls.matthewm.boedicker.org/
+s3_bucket: 
+s3_id: 
+s3_secret: 
 show_names: true
 thumbnail_max_side: 90
-user_agent: murlsh (http://github.com/mmb/murlsh)
+user_agent: murlsh (https://github.com/mmb/murlsh)

data/db/migrate/20110213023123_init.rb

@@ -0,0 +1,20 @@
+class Init < ActiveRecord::Migration
+
+  def self.up
+    create_table :urls do |t|
+      # t.integer :id
+      t.integer :content_length
+      t.string :content_type
+      t.string :email
+      t.string :name
+      t.timestamp :time
+      t.string :thumbnail_url
+      t.string :title
+      t.string :url
+      t.string :via
+    end
+
+    add_index :urls, :time
+  end
+
+end

data/lib/murlsh/atom_body.rb

@@ -0,0 +1,78 @@
+require 'uri'
+
+require 'tinyatom'
+
+require 'murlsh'
+
+module Murlsh
+
+  # Atom feed builder.
+  class AtomBody
+    include Murlsh::FeedBody
+
+    # Atom feed builder.
+    def build
+      if defined?(@body)
+        @body
+      else
+        feed = TinyAtom::Feed.new(config.fetch('root_url'), feed_title,
+          feed_url, :hubs => config.fetch('pubsubhubbub_hubs', []).
+          map { |x| x['subscribe_url'] })
+
+        urls.each do |mu|
+          Murlsh::Plugin.hooks('url_display_pre') do |p|
+            p.run mu, req, config
+          end
+
+          options = {
+            :author_name => mu.name,
+            :summary => mu.title_stripped
+          }
+
+          if EnclosureContentTypes.include?(mu.content_type)
+            options.merge!(
+              :enclosure_type => mu.content_type,
+              :enclosure_href => mu.url,
+              :enclosure_title => mu.title
+              )
+            if mu.content_length
+              options.merge! :enclosure_length => mu.content_length
+            end
+          end
+
+          if mu.thumbnail_url
+            begin
+              # Add root url to relative urls.
+              tu = URI(mu.thumbnail_url)
+              abs_url = if tu.is_a?(URI::HTTP)
+                tu
+              else
+                URI.join config.fetch('root_url'), tu
+              end
+              options.merge! :media_thumbnail_url => abs_url
+            rescue URI::InvalidURIError
+            end
+          end
+
+          Murlsh::failproof do
+            if mu.via
+              options.merge!(
+                :via_type => 'text/html',
+                :via_href => mu.via,
+                :via_title => URI(mu.via).extend(Murlsh::URIDomain).domain
+                )
+            end
+          end
+
+          feed.add_entry mu.id, mu.title_stripped, mu.time, mu.url, options
+        end
+
+        @updated = feed.updated
+        @body = feed.make
+      end
+
+    end
+
+  end
+
+end