murlsh 0.11.0 → 1.0.0

data/.htaccess

@@ -1,6 +1,8 @@
 Options -Indexes
 
+AddOutputFilterByType DEFLATE application/atom+xml
 AddOutputFilterByType DEFLATE application/javascript
+AddOutputFilterByType DEFLATE application/rss+xml
 AddOutputFilterByType DEFLATE application/xhtml+xml
 AddOutputFilterByType DEFLATE application/xml
 AddOutputFilterByType DEFLATE text/css

data/README.textile

@@ -1,7 +1,7 @@
 Site for sharing and archiving links.
 
 * looks up url titles
-* adds thumbnails for and jGrowls embedded versions of Flickr, Imageshack, Vimeo and YouTube urls
+* adds thumbnails for and jGrowls embedded versions of Imageshack, Vimeo and YouTube urls
 * converts Twitter status urls to their full text and adds user thumbnail
 * generates Atom and RSS feeds
 * regex search
@@ -12,10 +12,6 @@ Site for sharing and archiving links.
 * rack interface
 * Gravatar support
 
-!http://static.mmb.s3.amazonaws.com/murlsh_screenshot.jpg!
-
-!http://static.mmb.s3.amazonaws.com/murlsh_iphone_screenshot.jpg!
-
 See "http://urls.matthewm.boedicker.org/":http://urls.matthewm.boedicker.org/ for example.
 
 h1. Installation
@@ -59,11 +55,9 @@ Plugin hooks
 
 |Hook|Description|run() arguments|Returns|
 |add_pre|called before a new url is saved|url, config hash|undefined|
-|add_post|called after a new url is saved|config hash|undefined|
-|hostrec|post process the domain that is shown after links|domain, url, title|text to display|
+|add_post|called after a new url is saved|url, config hash|undefined|
 |html_parse|parse HTML using something like Hpricot or Nokogiri|parseable|parsed HTML, only first plugin is run (cannot be chained)|
-|time|convert the time of a post into a string for display|time|time display text|
-|via|convert a via url into a string for display|via url|via url display text|
+|url_display_add|called to display additional information after urls|markup builder, url, config hash|undefined|
 
 h1. PubSubHubbub
 
@@ -84,33 +78,9 @@ subscribe_url is what gets put in the feed as link rel="hub"
 
 This will make updates to your feed show up in Google Reader instantly.
 
-h1. Thumbnail Locators
-
-If the url for a thumbnail image can be generated by regex search and replace
-on a posted url, a rule can be added to the configuration to automatically
-show a thumbnail image for it. The 'thumb_locators' config key is a hash
-of Javascript regex to Javascript replacement string (which can include
-captured groups). The hash key is wrapped in ^ and $ before compilation so
-it must match the entire url. The regex is compiled case-insensitive.
-
-For example:
-
-* Show the GitHub octocat Apple touch icon
-(http://github.com/apple-touch-icon.png) as a thumbnail for GitHub links
-* Show a friend's Gravatar for the thumbnails of links to his blog
-
-<pre>
-<code>
-thumb_locators:
-  (http:\/\/github\.com\/).*: $1apple-touch-icon.png
-  http:\/\/myfriend\.com\/.*: http://gravatar.com/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx</code>
-</pre>
-
 h1. Design Goals
 
-* make site fast and very cacheable, caching wrapper around calls to third party JSON APIs
 * low effort required to add a url, get metadata programatically instead of requiring user to specify
-* heavy lifting on the client side
 * allow customization with config and plugins
 * full regex search for finding saved urls
 * simple security (no sessions, no cookies)

data/Rakefile

@@ -81,8 +81,10 @@ namespace :db do
       email TEXT,
       name TEXT,
       title TEXT,
+      content_length INTEGER,
       content_type TEXT,
-      via TEXT);
+      via TEXT,
+      thumbnail_url TEXT);
       ")
   end
 
@@ -190,8 +192,8 @@ end
 
 namespace :validate do
 
-  desc 'Validate XHTML.'
-  task :xhtml do
+  desc 'Validate HTML.'
+  task :html do
     check_url = config['root_url']
     print "validating #{check_url} : "
     result = validate(check_url)
@@ -340,17 +342,22 @@ begin
       activerecord 2.3.4
       bcrypt-ruby 2.1.2
       builder 2.1.2
+      flickraw 0.8.3
+      flog 2.5.0
       hpricot 0.8.1
       htmlentities 4.2.0
       json 1.2.3
       push-notify 0.1.0
       rack 1.0.0
       rack-cache 0.5.2
+      rack-rewrite 1.0.2
       rack-throttle 0.3.0
       sqlite3-ruby 1.2.1
-      tinyatom 0.1.1
+      tinyatom 0.2.0
+      twitter 0.9.12
+      vimeo 1.2.2
       }.each_slice(2) { |g,v| gemspec.add_dependency(g, ">= #{v}") }
-
+      gemspec.add_dependency('rspec',  '~> 1.3')
   end
 rescue LoadError
   puts "Jeweler not available. Install it with: gem install jeweler"

data/VERSION

@@ -1 +1 @@
-0.11.0
+1.0.0

data/config.ru

@@ -4,6 +4,7 @@ $:.unshift(File.join(File.dirname(__FILE__), 'lib'))
 yaml
 
 rack/cache
+rack/rewrite
 rack/throttle
 
 murlsh
@@ -25,11 +26,16 @@ use Murlsh::EtagAddEncoding
 use Rack::Deflater
 use Murlsh::FarFutureExpires, :patterns => %r{\.gen\.(css|js)$}
 
-feed_path = URI.join(config.fetch('root_url'), config.fetch('feed_file')).path
-use Murlsh::MustRevalidate, :patterns => %r{^#{Regexp.escape(feed_path)}$}
+feed_url = URI.join(config.fetch('root_url'), config.fetch('feed_file'))
+use Murlsh::MustRevalidate, :patterns => %r{^#{Regexp.escape(feed_url.path)}$}
 
-use Rack::Static, :urls => %w{/css /js /swf}, :root => 'public'
-use Rack::Static, :urls => %w{/atom.xml /rss.xml}
+use Rack::Static, :urls => %w{/css /img /js /swf}, :root => 'public'
+use Rack::Static, :urls => %w{/atom.atom /podcast.rss /rss.rss}
+
+use Rack::Rewrite do
+  r301 '/atom.xml', feed_url.to_s
+  r301 '/rss.xml', URI.join(config.fetch('root_url'), 'rss.rss').to_s
+end
 
 # use Rack::Lint
 

data/config.yaml

@@ -2,18 +2,19 @@
 auth_file: murlsh_users
 cache_entitystore: file:tmp/cache/rack/body
 cache_metastore: file:tmp/cache/rack/meta
-config_js: 
-- thumb_locators
+config_js: []
+
 css_files: 
 - css/jquery.jgrowl.css
 - css/screen.css
 db_file: murlsh.db
-feed_file: atom.xml
+feed_file: atom.atom
 flickr_api_key: 
 gravatar_size: 32
 js_files: 
-- js/jquery-1.4.2.min.js
+- js/jquery-1.4.3.min.js
 - js/jquery.jgrowl_compressed.js
+- js/twitter-text-1.0.3.js
 - js/js.js
 meta_tag_description: URLs found interesting by Matthew M. Boedicker
 meta_tag_verify-v1: 
@@ -25,11 +26,4 @@ pubsubhubbub_hubs: []
 
 root_url: http://urls.matthewm.boedicker.org/
 show_names: true
-thumb_locators:
-  (http:\/\/(cgi\.)?ebay\.(com|co\.uk)\/).*: $1apple-touch-icon.png
-  (http:\/\/blog\.makezine\.com\/).*: $1apple-touch-icon.png
-  (http:\/\/en\.wikipedia\.org\/).*: $1apple-touch-icon.png
-  (http:\/\/github\.com\/).*: $1apple-touch-icon.png
-  (http:\/\/stackoverflow\.com\/).*: $1apple-touch-icon.png
-  (http:\/\/www\.nytimes\.com\/).*: $1apple-touch-icon.png
-  (http:\/\/www\.wired\.com\/).*: $1apple-touch-icon.png
+user_agent: murlsh (http://github.com/mmb/murlsh)

data/lib/murlsh/dispatch.rb

@@ -22,8 +22,6 @@ module Murlsh
 
       url_server = Murlsh::UrlServer.new(@config, db)
       config_server = Murlsh::ConfigServer.new(@config)
-      flickr_server = Murlsh::FlickrServer.new(@config)
-      twitter_server = Murlsh::TwitterServer.new
 
       root_path = URI(@config.fetch('root_url')).path
 
@@ -33,10 +31,6 @@ module Murlsh
         [%r{^POST #{root_path}(url)?$}, url_server.method(:post)],
         [%r{^HEAD #{root_path}config$}, config_server.method(:head)],
         [%r{^GET #{root_path}config$}, config_server.method(:get)],
-        [%r{^HEAD #{root_path}flickr$}, flickr_server.method(:head)],
-        [%r{^GET #{root_path}flickr$}, flickr_server.method(:get)],
-        [%r{^HEAD #{root_path}twitter/.+$}, twitter_server.method(:head)],
-        [%r{^GET #{root_path}twitter/.+$}, twitter_server.method(:get)],
       ]
     end
 

data/lib/murlsh/img_store.rb

@@ -0,0 +1,36 @@
+%w{
+cgi
+open-uri
+}.each { |m| require m }
+
+module Murlsh
+
+  # Fetch images from urls and store them locally.
+  class ImgStore
+
+    def initialize(storage_dir, options={})
+      @storage_dir = storage_dir
+      @user_agent = options[:user_agent]
+    end
+
+    # Build headers to send with request.
+    def headers
+      result = {}
+      result['User-Agent'] = @user_agent if @user_agent
+      result
+    end
+
+    # Fetch an image from a url and store it locally.
+    def store(url)
+      local_file = CGI.escape(url)
+      local_path = File.join(storage_dir, local_file)
+      open(url, headers) do |fin|
+        open(local_path, 'w') { |fout| fout.write(fin.read) }
+      end
+      local_file
+    end
+
+    attr_reader :storage_dir
+  end
+
+end

data/lib/murlsh/markup.rb

@@ -53,7 +53,6 @@ module Murlsh
         attrs = {
           :href => "#{options[:prefix]}#{href}",
           :rel => 'stylesheet',
-          :type => 'text/css',
         }
         attrs[:media] = options[:media] if options[:media]
         link(attrs)

data/lib/murlsh/plugin.rb

@@ -7,14 +7,10 @@ module Murlsh
   #   run arguments (url, config hash)
   # * add_post - called after a new url is saved
   #   run arguments (config hash)
-  # * hostrec - called to post process the domain that shows after links
-  #   run arguments (domain, url, title)
   # * html_parse - called to parse HTML using something like Hpricot or Nokogiri
   #   run arguments (parseable)
-  # * time - called to convert the time of a post into a string for display
-  #   run arguments (time)
-  # * via - called to convert a via url into a string for display
-  #   run arguments (via url)
+  # * url_display_add - called to display additional information after urls
+  #   run arguments (markup builder, url, config hash)
   class Plugin
 
     # Called when a plugin class inherits from this class (the way plugins

data/lib/murlsh/uri_ask.rb

@@ -14,33 +14,19 @@ module Murlsh
   # URI mixin.
   module UriAsk
 
-    # Get the content type.
+    # Get the content length.
     #
     # Options:
     # * :failproof - if true hide all exceptions and return empty string on failure
     # * :headers - hash of headers to send in request
-    def content_type(options={})
-      return @content_type if defined?(@content_type)
-      options[:headers] = default_headers.merge(options.fetch(:headers, {}))
-
-      content_type = ''
-      Murlsh::failproof(options) do
-        # try head first to save bandwidth
-        http = Net::HTTP.new(host, port)
-        http.use_ssl = (scheme == 'https')
-
-        resp = http.request_head(path_query, options[:headers])
+    def content_length(options={}); header('content-length', options); end
 
-        if Net::HTTPSuccess === resp
-          content_type = resp['content-type']
-        end
-
-        if not content_type or content_type.empty?
-          content_type = self.open(options[:headers]) { |f| f.content_type }
-        end
-      end
-      @content_type = content_type
-    end
+    # Get the content type.
+    #
+    # Options:
+    # * :failproof - if true hide all exceptions and return empty string on failure
+    # * :headers - hash of headers to send in request
+    def content_type(options={}); header('content-type', options); end
 
     # Get the HTML title.
     #
@@ -134,6 +120,64 @@ module Murlsh
       HTMLEntities.new.decode(Iconv.conv('utf-8', @charset, s))
     end
 
+    # Get the value of a response header.
+    #
+    # Options:
+    # * :failproof - if true hide all exceptions and return empty string on failure
+    # * :headers - hash of headers to send in request
+    def header(header_name, options={})
+      result = [*head_headers(options)[header_name]][0]
+      result = get_headers(options)[header_name] if !result or result.empty?
+      result || ''
+    end
+
+    # Get and cache response headers returned by HTTP HEAD for this URI.
+    #
+    # Return hash values are lists.
+    #
+    # Options:
+    # * :failproof - if true hide all exceptions and return empty hash on failure
+    # * :headers - hash of headers to send in request
+    def head_headers(options={})
+      return @head_headers if defined?(@head_headers)
+
+      request_headers = default_headers.merge(options.fetch(:headers, {}))
+
+      response_headers = {}
+      Murlsh::failproof(options) do
+        http = Net::HTTP.new(host, port)
+        http.use_ssl = (scheme == 'https')
+
+        resp = http.request_head(path_query, request_headers)
+
+        if Net::HTTPSuccess === resp
+          response_headers = resp.to_hash
+        end
+
+      end
+      @head_headers = response_headers
+    end
+
+    # Get and cache response headers returned by HTTP GET for this URI.
+    #
+    # Return hash values are single strings.
+    #
+    # Options:
+    # * :failproof - if true hide all exceptions and return empty hash on failure
+    # * :headers - hash of headers to send in request
+    def get_headers(options={})
+      return @get_headers if defined?(@get_headers)
+
+      request_headers = default_headers.merge(options.fetch(:headers, {}))
+
+      response_headers = {}
+      # use open-uri instead of Net::HTTP because it handles redirects
+      Murlsh::failproof(options) do
+        response_headers = self.open(request_headers) { |f| f.meta }
+      end
+      @get_headers = response_headers
+    end
+
   end
 
 end

data/lib/murlsh/url.rb

@@ -31,24 +31,6 @@ module Murlsh
         email and name and email == other.email and name == other.name
     end
 
-    # Return text showing what domain a link goes to.
-    def hostrec
-      domain = Murlsh::failproof { URI(url).domain }
-
-      domain = Murlsh::Plugin.hooks('hostrec').inject(domain) {
-        |result,plugin| plugin.run(result, url, title) }
-
-      yield domain if domain
-    end
-
-    # Yield the url that the url came from.
-    def viarec; Murlsh::failproof { yield URI(via) } if via; end
-
-    # Return true if this url is an image.
-    def is_image?
-      %w{image/gif image/jpeg image/png}.include?(content_type)
-    end
-
   end
 
 end

data/lib/murlsh/url_body.rb

@@ -8,9 +8,10 @@ module Murlsh
   class UrlBody < Builder::XmlMarkup
     include Murlsh::Markup
 
-    def initialize(config, db, req)
-      @config, @db, @req, @q = config, db, req, req.params['q']
-      super(:indent => @config['xhtml_indent'] || 0)
+    def initialize(config, db, req, content_type='text/html')
+      @config, @db, @req, @q, @content_type =
+        config, db, req, req.params['q'], content_type
+      super(:indent => @config['html_indent'] || 0)
     end
 
     # Fetch urls base on query string parameters.
@@ -33,14 +34,9 @@ module Murlsh
 
     # Url list page body builder.
     def each
-      instruct! :xml
-      declare! :DOCTYPE, :html, :PUBLIC, '-//W3C//DTD XHTML 1.1//EN',
-        'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd'
-
-      yield html(:xmlns => 'http://www.w3.org/1999/xhtml',
-        :'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
-        :'xsi:schemaLocation' => 'http://www.w3.org/MarkUp/SCHEMA/xhtml11.xsd',
-        :'xml:lang' => 'en') {
+      declare! :DOCTYPE, :html
+
+      yield html(:lang => 'en') {
         headd
         body {
           ul(:id => 'urls') {
@@ -59,22 +55,17 @@ module Murlsh
                     @config.fetch('show_names', false) and mu.name
                 end
 
+                if mu.thumbnail_url
+                  murlsh_img(:src => mu.thumbnail_url,
+                    :text => mu.title_stripped, :class => 'thumb')
+                end
+
                 a(mu.title_stripped, :href => mu.url, :class => 'm')
 
-                mu.hostrec do |hostrec|
-                  self.span(" [#{hostrec}]", :class => 'host')
-                end
-                mu.viarec do |via|
-                  display_via = Murlsh::Plugin.hooks('via').inject(
-                    via) { |result,plugin| plugin.run(result) }
-                  span(:class => 'via') {
-                    text!(' via '); a(display_via, :href => via)
-                  }
+                Murlsh::Plugin.hooks('url_display_add') do |p|
+                  p.run(self, mu, @config)
                 end
 
-                display_time = Murlsh::Plugin.hooks('time').inject(
-                  mu.time) { |result,plugin| plugin.run(result) }
-                span(", #{display_time}", :class => 'date') if display_time
                 last = mu
               }
             end
@@ -94,6 +85,7 @@ module Murlsh
     def headd
       head {
         titlee
+        meta :'http-equiv' => 'Content-Type', :content => @content_type
         metas(@config.select { |k,v| k =~ /^meta_tag_/ and v }.
           map { |k,v| [k.sub('meta_tag_', ''), v] })
         css(@config['css_compressed'] || @config['css_files'])

data/lib/murlsh/url_server.rb

@@ -20,16 +20,16 @@ module Murlsh
     # Respond to a GET request. Return a page of urls based on the query
     # string parameters.
     def get(req)
-      resp = Murlsh::XhtmlResponse.new
+      last_db_update = File::Stat.new(@config['db_file']).mtime
 
-      resp.set_content_type(req.env['HTTP_ACCEPT'], req.env['HTTP_USER_AGENT'])
+      resp = Rack::Response.new
 
-      last_db_update = File::Stat.new(@config['db_file']).mtime
       resp['Cache-Control'] = 'must-revalidate, max-age=0'
+      resp['Content-Type'] = 'text/html; charset=utf-8'
       resp['ETag'] = "W/\"#{last_db_update.to_i}#{req.params.sort}\""
       resp['Last-Modified'] = last_db_update.httpdate
 
-      resp.body = Murlsh::UrlBody.new(@config, @db, req)
+      resp.body = Murlsh::UrlBody.new(@config, @db, req, resp['Content-Type'])
 
       resp
     end
@@ -55,7 +55,7 @@ module Murlsh
           raise ActiveRecord::RecordInvalid.new(mu) unless mu.valid?
           Murlsh::Plugin.hooks('add_pre') { |p| p.run(mu, @config) }
           mu.save!
-          Murlsh::Plugin.hooks('add_post') { |p| p.run(@config) }
+          Murlsh::Plugin.hooks('add_post') { |p| p.run(mu, @config) }
           response_body, response_code = [mu], 200
         rescue ActiveRecord::RecordInvalid => error
           response_body = {