murlsh 0.11.0 → 1.0.0

data/murlsh.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{murlsh}
-  s.version = "0.11.0"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matthew M. Boedicker"]
-  s.date = %q{2010-09-21}
+  s.date = %q{2010-11-03}
   s.default_executable = %q{murlsh}
   s.description = %q{url sharing site framework with easy adding, title lookup, atom feed, thumbnails and embedding}
   s.email = %q{matthewm@boedicker.org}
@@ -35,51 +35,59 @@ Gem::Specification.new do |s|
      "lib/murlsh/etag_add_encoding.rb",
      "lib/murlsh/failproof.rb",
      "lib/murlsh/far_future_expires.rb",
-     "lib/murlsh/flickr_server.rb",
      "lib/murlsh/head_from_get.rb",
+     "lib/murlsh/img_store.rb",
      "lib/murlsh/markup.rb",
      "lib/murlsh/must_revalidate.rb",
      "lib/murlsh/openlock.rb",
      "lib/murlsh/plugin.rb",
      "lib/murlsh/sqlite3_adapter.rb",
      "lib/murlsh/time_ago.rb",
-     "lib/murlsh/twitter_server.rb",
-     "lib/murlsh/unwrap_jsonp.rb",
      "lib/murlsh/uri.rb",
      "lib/murlsh/uri_ask.rb",
      "lib/murlsh/url.rb",
      "lib/murlsh/url_body.rb",
      "lib/murlsh/url_server.rb",
-     "lib/murlsh/xhtml_response.rb",
      "lib/murlsh/yaml_ordered_hash.rb",
      "murlsh.gemspec",
      "plugins/add_post_50_update_feed.rb",
+     "plugins/add_post_50_update_podcast.rb",
      "plugins/add_post_50_update_rss.rb",
      "plugins/add_post_60_notify_hubs.rb",
+     "plugins/add_pre_40_convert_mobile.rb",
      "plugins/add_pre_50_lookup_content_type_title.rb",
+     "plugins/add_pre_60_flickr.rb",
      "plugins/add_pre_60_github_title.rb",
      "plugins/add_pre_60_google_code_title.rb",
-     "plugins/hostrec_50_redundant.rb",
-     "plugins/hostrec_60_skip.rb",
+     "plugins/add_pre_60_imageshack.rb",
+     "plugins/add_pre_60_imgur.rb",
+     "plugins/add_pre_60_s3_image.rb",
+     "plugins/add_pre_60_twitter.rb",
+     "plugins/add_pre_60_vimeo.rb",
+     "plugins/add_pre_60_youtube.rb",
      "plugins/html_parse_50_hpricot.rb",
-     "plugins/time_50_ago.rb",
-     "plugins/via_50_domain.rb",
+     "plugins/url_display_add_45_mp3.rb",
+     "plugins/url_display_add_50_hostrec.rb",
+     "plugins/url_display_add_55_content_type.rb",
+     "plugins/url_display_add_60_via.rb",
+     "plugins/url_display_add_65_time.rb",
      "public/css/jquery.jgrowl.css",
      "public/css/screen.css",
+     "public/img/thumb/README",
      "public/js/comments.json",
-     "public/js/jquery-1.4.2.min.js",
+     "public/js/jquery-1.4.3.min.js",
      "public/js/jquery.jgrowl_compressed.js",
      "public/js/js.js",
+     "public/js/twitter-text-1.0.3.js",
      "public/swf/player_mp3_mini.swf",
      "spec/auth_spec.rb",
      "spec/dispatch_spec.rb",
      "spec/doc_spec.rb",
+     "spec/img_store_spec.rb",
      "spec/markup_spec.rb",
-     "spec/unwrap_json_spec.rb",
      "spec/uri_ask_spec.rb",
      "spec/uri_spec.rb",
      "spec/url_spec.rb",
-     "spec/xhtml_response_spec.rb",
      "spec/yaml_ordered_hash_spec.rb"
   ]
   s.homepage = %q{http://github.com/mmb/murlsh}
@@ -88,11 +96,10 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{url sharing site framework}
   s.test_files = [
-    "spec/xhtml_response_spec.rb",
-     "spec/auth_spec.rb",
+    "spec/auth_spec.rb",
      "spec/yaml_ordered_hash_spec.rb",
-     "spec/unwrap_json_spec.rb",
      "spec/uri_ask_spec.rb",
+     "spec/img_store_spec.rb",
      "spec/dispatch_spec.rb",
      "spec/markup_spec.rb",
      "spec/url_spec.rb",
@@ -108,42 +115,60 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<bcrypt-ruby>, [">= 2.1.2"])
       s.add_runtime_dependency(%q<builder>, [">= 2.1.2"])
+      s.add_runtime_dependency(%q<flickraw>, [">= 0.8.3"])
+      s.add_runtime_dependency(%q<flog>, [">= 2.5.0"])
       s.add_runtime_dependency(%q<hpricot>, [">= 0.8.1"])
       s.add_runtime_dependency(%q<htmlentities>, [">= 4.2.0"])
       s.add_runtime_dependency(%q<json>, [">= 1.2.3"])
       s.add_runtime_dependency(%q<push-notify>, [">= 0.1.0"])
       s.add_runtime_dependency(%q<rack>, [">= 1.0.0"])
       s.add_runtime_dependency(%q<rack-cache>, [">= 0.5.2"])
+      s.add_runtime_dependency(%q<rack-rewrite>, [">= 1.0.2"])
       s.add_runtime_dependency(%q<rack-throttle>, [">= 0.3.0"])
       s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 1.2.1"])
-      s.add_runtime_dependency(%q<tinyatom>, [">= 0.1.1"])
+      s.add_runtime_dependency(%q<tinyatom>, [">= 0.2.0"])
+      s.add_runtime_dependency(%q<twitter>, [">= 0.9.12"])
+      s.add_runtime_dependency(%q<vimeo>, [">= 1.2.2"])
+      s.add_runtime_dependency(%q<rspec>, ["~> 1.3"])
     else
       s.add_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_dependency(%q<bcrypt-ruby>, [">= 2.1.2"])
       s.add_dependency(%q<builder>, [">= 2.1.2"])
+      s.add_dependency(%q<flickraw>, [">= 0.8.3"])
+      s.add_dependency(%q<flog>, [">= 2.5.0"])
       s.add_dependency(%q<hpricot>, [">= 0.8.1"])
       s.add_dependency(%q<htmlentities>, [">= 4.2.0"])
       s.add_dependency(%q<json>, [">= 1.2.3"])
       s.add_dependency(%q<push-notify>, [">= 0.1.0"])
       s.add_dependency(%q<rack>, [">= 1.0.0"])
       s.add_dependency(%q<rack-cache>, [">= 0.5.2"])
+      s.add_dependency(%q<rack-rewrite>, [">= 1.0.2"])
       s.add_dependency(%q<rack-throttle>, [">= 0.3.0"])
       s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.1"])
-      s.add_dependency(%q<tinyatom>, [">= 0.1.1"])
+      s.add_dependency(%q<tinyatom>, [">= 0.2.0"])
+      s.add_dependency(%q<twitter>, [">= 0.9.12"])
+      s.add_dependency(%q<vimeo>, [">= 1.2.2"])
+      s.add_dependency(%q<rspec>, ["~> 1.3"])
     end
   else
     s.add_dependency(%q<activerecord>, [">= 2.3.4"])
     s.add_dependency(%q<bcrypt-ruby>, [">= 2.1.2"])
     s.add_dependency(%q<builder>, [">= 2.1.2"])
+    s.add_dependency(%q<flickraw>, [">= 0.8.3"])
+    s.add_dependency(%q<flog>, [">= 2.5.0"])
     s.add_dependency(%q<hpricot>, [">= 0.8.1"])
     s.add_dependency(%q<htmlentities>, [">= 4.2.0"])
     s.add_dependency(%q<json>, [">= 1.2.3"])
     s.add_dependency(%q<push-notify>, [">= 0.1.0"])
     s.add_dependency(%q<rack>, [">= 1.0.0"])
     s.add_dependency(%q<rack-cache>, [">= 0.5.2"])
+    s.add_dependency(%q<rack-rewrite>, [">= 1.0.2"])
     s.add_dependency(%q<rack-throttle>, [">= 0.3.0"])
     s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.1"])
-    s.add_dependency(%q<tinyatom>, [">= 0.1.1"])
+    s.add_dependency(%q<tinyatom>, [">= 0.2.0"])
+    s.add_dependency(%q<twitter>, [">= 0.9.12"])
+    s.add_dependency(%q<vimeo>, [">= 1.2.2"])
+    s.add_dependency(%q<rspec>, ["~> 1.3"])
   end
 end
 

data/plugins/add_post_50_update_feed.rb

@@ -1,4 +1,6 @@
 %w{
+uri
+
 tinyatom
 
 murlsh
@@ -11,7 +13,16 @@ module Murlsh
 
     @hook = 'add_post'
 
-    def self.run(config)
+    # content types to add an enclosure for
+    EnclosureContentTypes = %w{
+      application/pdf
+      audio/mpeg
+      image/gif
+      image/jpeg
+      image/png
+      }
+
+    def self.run(url, config)
       feed = TinyAtom::Feed.new(config['root_url'], config.fetch('page_title'),
         URI.join(config['root_url'], config['feed_file']),
         :hubs => config.fetch('pubsubhubbub_hubs', []).
@@ -26,12 +37,15 @@ module Murlsh
           :summary => mu.title_stripped
         }
 
-        if mu.is_image?
+        if EnclosureContentTypes.include?(mu.content_type)
           options.merge!(
             :enclosure_type => mu.content_type,
             :enclosure_href => mu.url,
-            :enclosure_title => 'Full-size'
+            :enclosure_title => mu.title
             )
+          if mu.content_length
+            options.merge!(:enclosure_length => mu.content_length)
+          end
         end
 
         Murlsh::failproof do

data/plugins/add_post_50_update_podcast.rb

@@ -0,0 +1,46 @@
+%w{
+rss/maker
+uri
+
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+  # Regenerate podcast RSS feed after a new audio/mpeg url has been added.
+  class AddPost50UpdatePodcast < Plugin
+
+    @hook = 'add_post'
+
+    def self.run(url, config)
+      if url.content_type == 'audio/mpeg'
+        output_file = 'podcast.rss'
+
+        feed = RSS::Maker.make('2.0') do |f|
+          f.channel.title = f.channel.description = config.fetch(
+            'page_title', '')
+          f.channel.link = URI.join(config.fetch('root_url'), output_file)
+          f.items.do_sort = true
+  
+          Murlsh::Url.all(:conditions => { :content_type => 'audio/mpeg' },
+            :order => 'id DESC',
+            :limit => config.fetch('num_posts_feed', 25)).each do |mu|
+            i = f.items.new_item
+            i.title = mu.title_stripped
+            i.link = mu.url
+            i.date = mu.time
+
+            i.enclosure.url = mu.url
+            i.enclosure.type = mu.content_type
+            i.enclosure.length = mu.content_length
+          end
+
+        end
+
+        Murlsh::openlock(output_file, 'w') { |f| f.write(feed) }
+      end
+    end
+
+  end
+
+end

data/plugins/add_post_50_update_rss.rb

@@ -1,5 +1,6 @@
 %w{
 rss/maker
+uri
 
 murlsh
 }.each { |m| require m }
@@ -11,8 +12,17 @@ module Murlsh
 
     @hook = 'add_post'
 
-    def self.run(config)
-      output_file = 'rss.xml'
+    # content types to add an enclosure for
+    EnclosureContentTypes = %w{
+      application/pdf
+      audio/mpeg
+      image/gif
+      image/jpeg
+      image/png
+      }
+
+    def self.run(url, config)
+      output_file = 'rss.rss'
 
       feed = RSS::Maker.make('2.0') do |f|
         f.channel.title = f.channel.description = config.fetch('page_title', '')
@@ -25,6 +35,12 @@ module Murlsh
           i.title = mu.title_stripped
           i.link = mu.url
           i.date = mu.time
+
+          if EnclosureContentTypes.include?(mu.content_type)
+            i.enclosure.url = mu.url
+            i.enclosure.type = mu.content_type
+            i.enclosure.length = mu.content_length
+          end
         end
 
       end

data/plugins/add_post_60_notify_hubs.rb

@@ -1,4 +1,6 @@
 %w{
+uri
+
 murlsh
 }.each { |m| require m }
 
@@ -9,7 +11,7 @@ module Murlsh
 
     @hook = 'add_post'
 
-    def self.run(config)
+    def self.run(url, config)
       hubs = config.fetch('pubsubhubbub_hubs', [])
 
       unless hubs.empty?

data/plugins/add_pre_40_convert_mobile.rb

@@ -0,0 +1,30 @@
+%w{
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+
+  # Convert urls specifically for use on mobile devices into their non-mobile
+  # equivalents.
+  class AddPre40ConvertMobile < Plugin
+
+    @hook = 'add_pre'
+
+    TwitterRe = %r{^(http://)mobile\.(twitter\.com/.*)$}i
+    WikipediaRe = %r{^(http://[a-z]+\.)m\.(wikipedia\.org/.*)$}i
+
+    def self.run(url, config)
+      url.url = case
+        when match = TwitterRe.match(url.url)
+          "#{match[1]}#{match[2]}"
+        when match = WikipediaRe.match(url.url)
+          "#{match[1]}#{match[2]}"
+        else
+          url.url
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_50_lookup_content_type_title.rb

@@ -1,19 +1,26 @@
 %w{
+uri
+
 murlsh
 }.each { |m| require m }
 
 module Murlsh
 
-  # try to fetch the content type and title of a url
+  # try to fetch the content length, content type and title of a url
   class AddPre50LookupContentTypeTitle < Plugin
 
     @hook = 'add_pre'
 
     def self.run(url, config)
       ask = URI(url.url).extend(Murlsh::UriAsk)
-      headers = {
-        'User-Agent' => 'murlsh (http://github.com/mmb/murlsh)'
-      }
+      headers = {}
+      headers['User-Agent'] = config['user_agent'] if config['user_agent']
+
+      content_length = ask.content_length(:headers => headers)
+      if content_length and not content_length.empty?
+        url.content_length = content_length
+      end
+
       url.content_type = ask.content_type(:headers => headers)
       url.title = ask.title(:headers => headers)
     end

data/plugins/add_pre_60_flickr.rb

@@ -0,0 +1,38 @@
+%w{
+cgi
+
+flickraw
+
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+  # Set the title and thumbnail url of Flickr photos.
+  class AddPre60Flickr < Plugin
+
+    @hook = 'add_pre'
+
+    FlickrRe = %r{^http://(?:www\.)?flickr\.com/photos/[@\w\-]+?/([\d]+)}i
+    StorageDir = File.join(File.dirname(__FILE__), '..', 'public', 'img',
+      'thumb')
+
+    def self.run(url, config)
+      if config['flickr_api_key'] and not config['flickr_api_key'].empty?
+        if photo_id = url.url[FlickrRe, 1]
+          FlickRaw.api_key = config['flickr_api_key']
+          info = flickr.photos.getInfo(:photo_id => photo_id)
+
+          url.title = "#{info.title} by #{info.owner.username}"
+
+          thumb_storage = Murlsh::ImgStore.new(StorageDir,
+            :user_agent => config['user_agent'])
+          stored_filename = thumb_storage.store(FlickRaw.url_s(info))
+          url.thumbnail_url = "img/thumb/#{CGI.escape(stored_filename)}"
+        end
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_60_github_title.rb

@@ -1,4 +1,6 @@
 %w{
+uri
+
 murlsh
 }.each { |m| require m }
 
@@ -10,8 +12,10 @@ module Murlsh
 
     @hook = 'add_pre'
 
+    GithubRe = %r{^http://github\.com/\w+/[\w.-]+$}i
+
     def self.run(url, config)
-      if url.url[%r{^http://github\.com/\w+/[\w.-]+$}]
+      if url.url[GithubRe]
         ask = URI(url.url).extend(Murlsh::UriAsk)
         url.title << " - #{ask.description}" unless ask.description.empty?
       end

data/plugins/add_pre_60_google_code_title.rb

@@ -1,4 +1,6 @@
 %w{
+uri
+
 murlsh
 }.each { |m| require m }
 
@@ -10,9 +12,10 @@ module Murlsh
 
     @hook = 'add_pre'
 
+    GoogleCodeRe = %r{^http://code\.google\.com/p/[\w-]+/$}i
+
     def self.run(url, config)
-      if url.url[%r{^http://code\.google\.com/p/[\w-]+/$}]
-        puts 'xxx'
+      if url.url[GoogleCodeRe]
         ask = URI(url.url).extend(Murlsh::UriAsk)
         ask.doc.xpath_search("//a[@id='project_summary_link']") do |node|
           summary = node ? node.inner_html : nil

data/plugins/add_pre_60_imageshack.rb

@@ -0,0 +1,31 @@
+%w{
+cgi
+
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+  # Set the thumbnail of imageshack images.
+  class AddPre60Imageshack < Plugin
+
+    @hook = 'add_pre'
+
+    ImageshackRe =
+      %r{^(http://img\d+\.imageshack\.us/img\d+/\d+/)(\w+)\.(jpe?g|gif|png)$}i
+    StorageDir = File.join(File.dirname(__FILE__), '..', 'public', 'img',
+      'thumb')
+
+    def self.run(url, config)
+      if match = ImageshackRe.match(url.url)
+        thumb_storage = Murlsh::ImgStore.new(StorageDir,
+          :user_agent => config['user_agent'])
+        stored_filename = thumb_storage.store(
+          "#{match[1]}#{match[2]}.th.#{match[3]}")
+        url.thumbnail_url = "img/thumb/#{CGI.escape(stored_filename)}"
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_60_imgur.rb

@@ -0,0 +1,32 @@
+%w{
+cgi
+
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+  # Add YouTube thumbnail.
+  class AddPre60Imgur < Plugin
+
+    @hook = 'add_pre'
+
+    ImgurRe = %r{^(http://(?:i\.)?imgur\.com/)([a-z\d]+)(\.(?:jpe?g|gif|png))$}i
+    StorageDir = File.join(File.dirname(__FILE__), '..', 'public', 'img',
+      'thumb')
+
+    def self.run(url, config)
+      if match = ImgurRe.match(url.url)
+        url.title = "imgur/#{match[2]}s#{match[3]}"
+
+        thumb_storage = Murlsh::ImgStore.new(StorageDir,
+          :user_agent => config['user_agent'])
+        stored_filename = thumb_storage.store(
+          "#{match[1]}#{match[2]}s#{match[3]}")
+        url.thumbnail_url = "img/thumb/#{CGI.escape(stored_filename)}"
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_60_s3_image.rb

@@ -0,0 +1,34 @@
+%w{
+cgi
+
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+  # Set the thumbnail of s3 hosted images.
+  class AddPre60S3Image < Plugin
+
+    @hook = 'add_pre'
+
+    S3ImageRe = %r{^(http://static\.mmb\.s3\.amazonaws\.com/)([\w\-]+)\.(jpe?g|gif|pdf|png)$}i
+    StorageDir = File.join(File.dirname(__FILE__), '..', 'public', 'img',
+      'thumb')
+
+    def self.run(url, config)
+      if match = S3ImageRe.match(url.url)
+        extension = match[3].downcase == 'pdf' ? 'png' : match[3]
+
+        thumb_storage = Murlsh::ImgStore.new(StorageDir,
+          :user_agent => config['user_agent'])
+        stored_filename = thumb_storage.store(
+          "#{match[1]}#{match[2]}.th.#{extension}")
+
+        url.thumbnail_url = "img/thumb/#{CGI.escape(stored_filename)}"
+        url.title = match[2]
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_60_twitter.rb

@@ -0,0 +1,35 @@
+%w{
+cgi
+
+twitter
+
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+  # Set title to entire tweet and set thumbnail url.
+  class AddPre60Twitter < Plugin
+
+    @hook = 'add_pre'
+
+    TwitterRe = %r{^https?://twitter\.com/\w+/status(?:es)?/(\d+)$}i
+    StorageDir = File.join(File.dirname(__FILE__), '..', 'public', 'img',
+      'thumb')
+
+    def self.run(url, config)
+      if tweet_id = url.url[TwitterRe, 1]
+        tweet = Twitter.status(tweet_id)
+
+        url.title = "@#{tweet.user.screen_name}: #{tweet.text}"
+
+        thumb_storage = Murlsh::ImgStore.new(StorageDir,
+          :user_agent => config['user_agent'])
+        stored_filename = thumb_storage.store(tweet.user.profile_image_url)
+        url.thumbnail_url = "img/thumb/#{CGI.escape(stored_filename)}"
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_60_vimeo.rb

@@ -0,0 +1,35 @@
+%w{
+cgi
+
+vimeo
+
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+  # Set title and thumbnail url for Vimeo urls.
+  class AddPre60Vimeo < Plugin
+
+    @hook = 'add_pre'
+
+    VimeoRe = %r{^http://(?:www\.)?vimeo\.com/(\d+)$}i
+    StorageDir = File.join(File.dirname(__FILE__), '..', 'public', 'img',
+      'thumb')
+
+    def self.run(url, config)
+      if id = url.url[VimeoRe, 1]
+        info = Vimeo::Simple::Video.info(id)[0]
+
+        url.title = "#{info['title']} by #{info['user_name']}"
+
+        thumb_storage = Murlsh::ImgStore.new(StorageDir,
+          :user_agent => config['user_agent'])
+        stored_filename = thumb_storage.store(info['thumbnail_small'])
+        url.thumbnail_url = "img/thumb/#{CGI.escape(stored_filename)}"
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_60_youtube.rb

@@ -0,0 +1,31 @@
+%w{
+cgi
+
+murlsh
+}.each { |m| require m }
+
+module Murlsh
+
+  # Add YouTube thumbnail.
+  class AddPre60Youtube < Plugin
+
+    @hook = 'add_pre'
+
+    YoutubeRe =
+      %r{^http://(?:(?:www|uk)\.)?youtube\.com/watch\?v=([\w\-]+)(?:&|$)}i
+    StorageDir = File.join(File.dirname(__FILE__), '..', 'public', 'img',
+      'thumb')
+
+    def self.run(url, config)
+      if youtube_id = url.url[YoutubeRe, 1]
+        thumb_storage = Murlsh::ImgStore.new(StorageDir,
+          :user_agent => config['user_agent'])
+        stored_filename = thumb_storage.store(
+          "http://img.youtube.com/vi/#{youtube_id}/default.jpg")
+        url.thumbnail_url = "img/thumb/#{CGI.escape(stored_filename)}"
+      end
+    end
+
+  end
+
+end

data/plugins/html_parse_50_hpricot.rb

@@ -1,5 +1,7 @@
 %w{
 hpricot
+
+murlsh
 }.each { |m| require m }
 
 module Murlsh