murlsh 1.8.0 → 1.9.0

data/ChangeLog

@@ -1,3 +1,20 @@
+1.9.0
+	2011-09-25
+
+	Do not strip trailing slashes from urls during normalization.
+
+	Add enclosures to JSON.
+
+	Support multiple enclosures per url that can be set by plugins.
+
+	Plugins for adding enclosures for YouTube and Vimeo urls.
+
+	Support builder version 3.
+
+	Update user agent sent when asking about urls to more recent browser.
+
+	Use "required" instead of "optional" attribute for form inputs.
+
 1.8.0
 	2011-06-22
 

data/db/migrate/20110906003222_create_enclosures.rb

@@ -0,0 +1,32 @@
+require 'murlsh'
+
+class CreateEnclosures < ActiveRecord::Migration
+
+  def self.up
+    create_table :enclosures do |t|
+      t.integer :content_length
+      t.string  :content_type
+      t.string  :title
+      t.string  :enclosure_url
+
+      t.references :url
+    end
+
+    add_index :enclosures, :url_id
+
+    # populate
+    require './plugins/add_pre_55_enclosure_self.rb'
+    require './plugins/add_pre_55_enclosure_vimeo.rb'
+    require './plugins/add_pre_55_enclosure_youtube.rb'
+
+    Murlsh::Url.find(:all).each do |u|
+      Murlsh::Plugin.hooks('add_pre') { |p| p.run u, {} }
+      u.save!
+    end
+  end
+
+  def self.down
+    drop_table :enclosures
+  end
+
+end

data/lib/murlsh/atom_body.rb

@@ -29,15 +29,17 @@ module Murlsh
             :summary => mu.title_stripped
           }
 
-          if EnclosureContentTypes.include?(mu.content_type)
-            options.merge!(
-              :enclosure_type => mu.content_type,
-              :enclosure_href => mu.url,
-              :enclosure_title => mu.title
-              )
-            if mu.content_length
-              options.merge! :enclosure_length => mu.content_length
-            end
+          options[:enclosures] = []
+          mu.enclosures.each do |e|
+            new_e = {
+              :enclosure_href => e.enclosure_url,
+              :enclosure_title => e.title_stripped,
+              :enclosure_type => e.content_type
+            }
+
+            new_e[:enclosure_length] = e.content_length  if e.content_length
+
+            options[:enclosures] << new_e
           end
 
           if mu.thumbnail_url

data/lib/murlsh/enclosure.rb

@@ -0,0 +1,14 @@
+require 'active_record'
+
+module Murlsh
+
+  # Enclosure ActiveRecord.
+  class Enclosure < ActiveRecord::Base
+    belongs_to :url
+
+    # Title with whitespace compressed and leading and trailing whitespace
+    # stripped.
+    def title_stripped; title.to_s.strip.gsub(/\s+/, ' '); end
+  end
+
+end

data/lib/murlsh/feed_body.rb

@@ -3,15 +3,6 @@ module Murlsh
   # Feed body mixin.
   module FeedBody
 
-    # Content types to add an enclosure for.
-    EnclosureContentTypes = %w{
-      application/pdf
-      audio/mpeg
-      image/gif
-      image/jpeg
-      image/png
-      }
-
     def initialize(config, req, feed_url, urls)
       @config, @req, @feed_url, @urls = config, req, feed_url, urls
       @updated = nil

data/lib/murlsh/json_body.rb

@@ -33,6 +33,12 @@ module Murlsh
               h['thumbnail_url']).to_s
           end
 
+          h['enclosures'] = mu.enclosures.map do |e|
+            new_e = e.attributes.reject { |k,v| k == 'url_id' }
+            new_e['title'] = e.title_stripped
+            new_e
+          end
+
           h
         end
         @body = urls.to_json

data/lib/murlsh/markup.rb

@@ -8,7 +8,7 @@ module Murlsh
     # Options:
     # * :prefix - prefix to append to all script urls
     def javascript(sources, options={})
-      Array(sources).each do |src|
+      ::Kernel::Array(sources).each do |src|
         script '', :type => 'text/javascript',
           :src => "#{options[:prefix]}#{src}"
       end
@@ -50,7 +50,7 @@ module Murlsh
     # * :media - optional media attribute
     # * :prefix - prepended to all CSS urls
     def css(hrefs, options={})
-      Array(hrefs).each do |href|
+      ::Kernel::Array(hrefs).each do |href|
         attrs = {
           :href => "#{options[:prefix]}#{href}",
           :rel => 'stylesheet',
@@ -69,7 +69,7 @@ module Murlsh
     def form_input(options)
       if options[:id]
         if options[:label]
-          label_class = options[:optional] ? 'optional' : 'required'
+          label_class = options[:required] ? 'required' : 'optional'
           label options[:label], :for => options[:id], :class => label_class
         end
         options[:name] ||= options[:id]

data/lib/murlsh/rss_body.rb

@@ -29,11 +29,12 @@ module Murlsh
             i.link = mu.url
             i.date = mu.time
 
-            if EnclosureContentTypes.include? mu.content_type
-              i.enclosure.url = mu.url
-              i.enclosure.type = mu.content_type
-              i.enclosure.length = mu.content_length
+            mu.enclosures.first(1).each do |e|
+              i.enclosure.url = e.enclosure_url
+              i.enclosure.type = e.content_type
+              i.enclosure.length = e.content_length
             end
+
           end
         end
 

data/lib/murlsh/uri_ask.rb

@@ -93,7 +93,7 @@ module Murlsh
     def default_headers
       result = {
         'User-Agent' =>
-          'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
+          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.126 Safari/535.1',
         }
       if host.to_s[/^www\.nytimes\.com/]
         result['Referer'] = 'http://news.google.com/'

data/lib/murlsh/url.rb

@@ -6,6 +6,7 @@ module Murlsh
 
   # URL ActiveRecord.
   class Url < ActiveRecord::Base
+    has_many :enclosures
     validates_format_of :url, :with => URI.regexp
 
     # Get the title of this url.
@@ -21,7 +22,7 @@ module Murlsh
 
     # Title with whitespace compressed and leading and trailing whitespace
     # stripped.
-    def title_stripped; title.strip.gsub(/\s+/, ' '); end
+    def title_stripped; title.to_s.strip.gsub(/\s+/, ' '); end
 
     # Return true if this url has the same author as another url.
     def same_author?(other)

data/lib/murlsh/url_body.rb

@@ -5,7 +5,7 @@ module Murlsh
 
   # Url list page builder.
   class UrlBody < Builder::XmlMarkup
-    include Murlsh::Markup
+    include ::Murlsh::Markup
 
     def initialize(config, req, result_set, content_type='text/html')
       @config, @req, @result_set, @content_type = config, req, result_set,
@@ -20,7 +20,7 @@ module Murlsh
       if page.to_i >= 1
         query = @req.params.dup
         query['p'] = page
-        Murlsh.build_query(query)
+        ::Murlsh.build_query(query)
       end
     end
 
@@ -50,13 +50,13 @@ module Murlsh
               last = nil
 
               @result_set.results.each do |mu|
-                Murlsh::Plugin.hooks('url_display_pre') do |p|
+                ::Murlsh::Plugin.hooks('url_display_pre') do |p|
                   p.run mu, @req, @config
                 end
 
                 li(:id => "liu#{mu.id}") {
                   unless mu.same_author?(last)
-                    avatar_url = Murlsh::Plugin.hooks('avatar').inject(
+                    avatar_url = ::Murlsh::Plugin.hooks('avatar').inject(
                       nil) do |url_so_far,plugin|
                       plugin.run url_so_far, mu, @config
                     end
@@ -74,7 +74,7 @@ module Murlsh
 
                   a mu.title_stripped, :href => mu.url, :class => 'm'
 
-                  Murlsh::Plugin.hooks('url_display_add') do |p|
+                  ::Murlsh::Plugin.hooks('url_display_add') do |p|
                     p.run self, mu, @config
                   end
 
@@ -154,7 +154,7 @@ module Murlsh
           order += (@config['quick_search'].keys - order).sort
           order.each do |k|
             if v = @config['quick_search'][k]
-              a "/#{k}", :href => "?q=#{URI.escape(v)}" ; text! ' '
+              a "/#{k}", :href => "?q=#{::URI.escape(v)}" ; text! ' '
             end
           end
         }
@@ -188,18 +188,15 @@ module Murlsh
     def add_form
       form(:action => 'url', :method => 'post') {
         fieldset(:id => 'add') {
-          self.p { form_input :id => 'url', :label => 'Add URL', :size => 32 }
           self.p {
-            form_input :id => 'title', :label => 'Title', :size => 32,
-              :optional => true
-          }
-          self.p {
-            form_input :id => 'via', :label => 'Via', :size => 32,
-              :optional => true
+            form_input :id => 'url', :label => 'Add URL', :size => 32,
+              :required => ''
           }
+          self.p { form_input :id => 'title', :label => 'Title', :size => 32 }
+          self.p { form_input :id => 'via', :label => 'Via', :size => 32 }
           self.p {
             form_input :type => 'password', :id => 'auth', :label => 'Password',
-              :size => 16
+              :size => 16, :required => ''
             form_input :id => 'submit', :type => 'button', :value => 'Add'
           }
         }

data/lib/murlsh/version.rb

@@ -1,3 +1,3 @@
 module Murlsh
-  VERSION = '1.8.0'
+  VERSION = '1.9.0'
 end

data/lib/murlsh.rb

@@ -11,6 +11,7 @@ require 'murlsh/delicious_parse'
 require 'murlsh/dispatch'
 require 'murlsh/doc'
 require 'murlsh/etag_add_encoding'
+require 'murlsh/enclosure'
 require 'murlsh/failproof'
 require 'murlsh/far_future_expires'
 require 'murlsh/feed_body'

data/plugins/add_pre_35_url_clean.rb

@@ -4,26 +4,49 @@ require 'postrank-uri'
 
 require 'murlsh'
 
+# Patch PostRank::URI.normalize to not strip trailing slashes from the path.
+module PostRank
+
+  module URI
+
+    module_function
+
+    def normalize(uri, opts = {})
+      u = parse(uri, opts)
+      u.path = u.path.squeeze('/')
+      # u.path = u.path.chomp('/') if u.path.size != 1
+      u.query = nil if u.query && u.query.empty?
+      u.fragment = nil
+      u
+    end
+
+  end
+
+end
+
 module Murlsh
 
-  # Canonicalize and clean urls with postrank-uri.
-  #
-  # See https://github.com/postrank-labs/postrank-uri
+  # Canonicalize and clean urls.
+
   class AddPre35UrlClean < Plugin
 
     @hook = 'add_pre'
 
     def self.run(url, config)
-      url.url = PostRank::URI.clean(url.url)  if cleanable?(url.url)
-      url.via = PostRank::URI.clean(url.via)  if cleanable?(url.via)
+      Murlsh::failproof { url.url = clean(url.url)  if cleanable?(url.url) }
+      Murlsh::failproof { url.via = clean(url.via)  if cleanable?(url.via) }
     end
 
-    # Return true if the url can be cleaned by postrank-uri.
+    # Canonicalize and clean a url using PostRank::URI.clean.
     #
-    # Postrank::URI.clean only works on https or https urls.
-    def self.cleanable?(url)
-      Murlsh::failproof { URI(url).scheme.to_s.match(/^https?$/i) }
-    end
+    # PostRank::URI.normalize is patched to not strip trailing slashes from
+    # the path (see above).
+    #
+    # See https://github.com/postrank-labs/postrank-uri
+    def self.clean(url); PostRank::URI.clean(url); end
+
+    # Return true if the url can be cleaned (is http or https).
+    def self.cleanable?(url); URI(url).scheme.to_s.match(/^https?$/i); end
 
   end
 

data/plugins/add_pre_55_enclosure_self.rb

@@ -0,0 +1,31 @@
+require 'murlsh'
+
+module Murlsh
+
+  # Create enclosures for content types that can directly become enclosures.
+  class AddPre55EnclosureSelf < Plugin
+
+    @hook = 'add_pre'
+
+    # Content types that can be enclosures.
+    ContentTypes = %w{
+      application/pdf
+      audio/mpeg
+      image/gif
+      image/jpeg
+      image/png
+      }
+
+    def self.run(url, config)
+      if ContentTypes.include?(url.content_type)
+        url.enclosures.build(
+          :content_length => url.content_length,
+          :title => url.title,
+          :content_type => url.content_type,
+          :enclosure_url => url.url)
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_55_enclosure_vimeo.rb

@@ -0,0 +1,24 @@
+require 'murlsh'
+
+module Murlsh
+
+  # Create video enclosures for Vimeo urls.
+  class AddPre55EnclosureVimeo < Plugin
+
+    @hook = 'add_pre'
+
+    VimeoRe = %r{^http://(?:www\.)?vimeo\.com/(\d+)$}i
+
+    def self.run(url, config)
+      if match = VimeoRe.match(url.url)
+        url.enclosures.build(
+          :title => url.title,
+          :content_type => 'application/x-shockwave-flash',
+          :enclosure_url =>
+            "http://vimeo.com/moogaloop.swf?clip_id=#{match[1]}")
+      end
+    end
+
+  end
+
+end

data/plugins/add_pre_55_enclosure_youtube.rb

@@ -0,0 +1,24 @@
+require 'murlsh'
+
+module Murlsh
+
+  # Create video enclosures for YouTube urls.
+  class AddPre55EnclosureYoutube < Plugin
+
+    @hook = 'add_pre'
+
+    YoutubeRe =
+      %r{^http://(?:(?:www|uk)\.)?youtube\.com/watch\?v=([\w\-]+)(?:&|$)}i
+
+    def self.run(url, config)
+      if match = YoutubeRe.match(url.url)
+        url.enclosures.build(
+          :title => url.title,
+          :content_type => 'application/x-shockwave-flash',
+          :enclosure_url => "http://www.youtube.com/v/#{match[1]}.swf")
+      end
+    end
+
+  end
+
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: murlsh
 version: !ruby/object:Gem::Version 
-  hash: 55
+  hash: 51
   prerelease: 
   segments: 
   - 1
-  - 8
+  - 9
   - 0
-  version: 1.8.0
+  version: 1.9.0
 platform: ruby
 authors: 
 - Matthew M. Boedicker
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-22 00:00:00 -04:00
+date: 2011-09-25 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -377,6 +377,7 @@ files:
 - config.yaml
 - config/README
 - db/migrate/20110213023123_init.rb
+- db/migrate/20110906003222_create_enclosures.rb
 - lib/murlsh.rb
 - lib/murlsh/ask.rb
 - lib/murlsh/atom_body.rb
@@ -388,6 +389,7 @@ files:
 - lib/murlsh/delicious_parse.rb
 - lib/murlsh/dispatch.rb
 - lib/murlsh/doc.rb
+- lib/murlsh/enclosure.rb
 - lib/murlsh/etag_add_encoding.rb
 - lib/murlsh/failproof.rb
 - lib/murlsh/far_future_expires.rb
@@ -433,6 +435,9 @@ files:
 - plugins/add_pre_50_lookup_content_type_title.rb
 - plugins/add_pre_50_media_thumbnail.rb
 - plugins/add_pre_50_open_graph_image.rb
+- plugins/add_pre_55_enclosure_self.rb
+- plugins/add_pre_55_enclosure_vimeo.rb
+- plugins/add_pre_55_enclosure_youtube.rb
 - plugins/add_pre_55_open_graph_title.rb
 - plugins/add_pre_60_github_title.rb
 - plugins/add_pre_60_imgur.rb