murlsh 0.4.0 → 0.5.0

data/Rakefile

@@ -21,7 +21,8 @@ config = YAML.load_file('config.yaml')
 
 desc "Test remote content type fetch for a URL and show errors."
 task :content_type, :url do |t, args|
-  puts Murlsh.get_content_type(args.url, :failproof => false, :debug => STDOUT)
+  puts URI(args.url).extend(Murlsh::UriAsk).content_type(:failproof => false,
+    :debug => STDOUT)
 end
 
 namespace :db do
@@ -101,7 +102,8 @@ end
 
 desc "Test remote title fetch for a URL and show errors."
 task :title, :url do |t, args|
-  puts Murlsh.get_title(args.url, :failproof => false, :debug => STDOUT)
+  puts URI(args.url).extend(Murlsh::UriAsk).title(:failproof => false,
+    :debug => STDOUT)
 end
 
 desc 'Try to fetch the title for a url and update it in the database.'
@@ -111,7 +113,7 @@ task :title_fetch, :url_id do |t, args|
   url = Murlsh::Url.find(args.url_id)
   puts "Url: #{url.url}"
   puts "Previous title: #{url.title}"
-  url.title = Murlsh.get_title(url.url, :failproof => false)
+  url.title = URI(url.url).extend(Murlsh::UriAsk).title(:failproof => false)
   url.save
   puts "\nNew title: #{url.title}"
 end

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0

data/config.yaml

@@ -13,7 +13,7 @@ css_files:
 - css/jquery.jgrowl.css
 - css/screen.css
 js_files:
-- js/jquery-1.3.2.min.js
+- js/jquery-1.4.min.js
 - js/jquery.cookie.js
 - js/jquery.jgrowl_compressed.js
 - js/js.js

data/lib/murlsh/atom_feed.rb

@@ -21,18 +21,9 @@ module Murlsh
       @filename = options[:filename]
       @title = options[:title]
 
-      setup_id_fields
-    end
-
-    # Set up fields to use for building item ids.
-    def setup_id_fields
-      uri_parsed = URI(@root_url)
-
-      m = uri_parsed.host.match(/^(.*?)\.?([^.]+\.[^.]+)$/)
+      root_uri = URI(@root_url)
 
-      @host, @domain = (m ? m.captures : [uri_parsed.host, ''])
-
-      @path = uri_parsed.path
+      @host, @domain, @path = root_uri.host, root_uri.domain, root_uri.path
     end
 
     # Generate the feed and write it to the filesystem with locking.
@@ -54,16 +45,16 @@ module Murlsh
         entries.each do |mu|
           xm.entry {
             xm.author { xm.name(mu.name) }
-            xm.title(mu.title)
+            xm.title(mu.title_stripped)
             xm.id(entry_id(mu))
-            xm.summary(mu.title)
+            xm.summary(mu.title_stripped)
             xm.updated(mu.time.xmlschema)
             xm.link(:href => mu.url)
             enclosure(xm, mu)
+            via(xm, mu)
           }
         end
       }
-      xm
     end
 
     # Build the entry's id.
@@ -77,6 +68,14 @@ module Murlsh
         :title => 'Full-size') if mu.is_image?
     end
 
+    def via(xm, mu)
+      begin
+        xm.link(:rel => 'via', :type => 'text/html', :href => mu.via,
+          :title => URI(mu.via).domain) if mu.via
+      rescue Exception
+      end
+    end
+
   end
 
 end

data/lib/murlsh/doc.rb

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'hpricot'
+
+module Murlsh
+
+  # Hpricot:Doc mixin.
+  module Doc
+
+    # Get the character set of the document.
+    def charset
+      %w{content-type Content-Type}.each do |ct|
+        content_type = at("meta[@http-equiv='#{ct}']")
+        unless content_type.nil?
+          content = content_type['content']
+          unless content.nil?
+            charset = content[/charset=([\w_.:-]+)/, 1]
+            return charset if charset
+          end
+        end
+      end
+      nil
+    end
+
+    # Find the title of the document.
+    def title
+      %w{//html/head/title //head/title //html/title //title}.each do |xpath|
+        return (self/xpath).first.inner_html unless (self/xpath).first.nil?
+      end
+      nil
+    end
+
+  end
+
+end

data/lib/murlsh/failproof.rb

@@ -0,0 +1,15 @@
+
+module Murlsh
+
+  module_function
+
+  # Catch all exceptions unless options[:failproof] = false.
+  def failproof(options={})
+    begin
+      yield
+    rescue Exception
+      raise unless options.fetch(:failproof, true)
+    end
+  end
+
+end

data/lib/murlsh/uri.rb

@@ -0,0 +1,12 @@
+require 'uri'
+
+# Extra methods added to URI class.
+class URI::Generic
+
+  # Return the domain.
+  def domain; host[/[a-z\d-]+\.[a-z]{2,}(\.[a-z]{2})?$/].downcase; end
+
+  # Return the path and query string.
+  def path_query; path + (query ? "?#{query}" : ''); end
+
+end

data/lib/murlsh/uri_ask.rb

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'net/https'
+require 'open-uri'
+require 'uri'
+
+require 'rubygems'
+require 'hpricot'
+require 'htmlentities'
+require 'iconv'
+
+module Murlsh
+
+  # URI mixin.
+  module UriAsk
+
+    # Get the content type.
+    #
+    # Options:
+    # * :failproof - if true hide all exceptions and return empty string on failure
+    # * :headers - hash of headers to send in request
+    def content_type(options={})
+      return @content_type if defined?(@content_type)
+      options[:headers] = default_headers.merge(options.fetch(:headers, {}))
+
+      @content_type = ''
+      Murlsh::failproof(options) do
+        # try head first to save bandwidth
+        http = Net::HTTP.new(host, port)
+        http.use_ssl = (scheme == 'https')
+
+        resp = http.request_head(path_query, options[:headers])
+        @content_type = case resp
+          when Net::HTTPSuccess then resp['content-type']
+          else self.open(options[:headers]) { |f| f.content_type }
+        end
+      end
+      @content_type
+    end
+
+    # Get the HTML title.
+    #
+    # Options:
+    # * :failproof - if true hide all exceptions and return empty string on failure
+    # * :headers - hash of headers to send in request
+    def title(options={})
+      return @title if defined?(@title)
+      options[:headers] = default_headers.merge(options.fetch(:headers, {}))
+
+      @title = to_s
+      if might_have_title?(options)
+        Murlsh::failproof(options) do
+          self.open(options[:headers]) do |f|
+            doc = Hpricot(f).extend(Murlsh::Doc)
+
+            @title = HTMLEntities.new.decode(Iconv.conv('utf-8',
+              doc.charset || f.charset, doc.title))
+          end
+        end
+      end
+      @title
+    end
+
+    # Default headers sent with the request.
+    def default_headers
+      result = {
+        'User-Agent' =>
+          'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
+        }
+      if (host || '')[/^www\.nytimes\.com/]
+        result['Referer'] = 'http://news.google.com/'
+      end
+
+      result
+    end
+
+    # Return true if the content type is likely to have a title that can be
+    # parsed.
+    def might_have_title?(options={})
+      content_type(options)[/^text\/html/]
+    end
+
+  end
+
+end

data/lib/murlsh/url.rb

@@ -3,10 +3,6 @@ require 'active_record'
 
 require 'uri'
 
-module URI
-  def domain; host[/[a-z\d-]+\.[a-z]{2,}(\.[a-z]{2})?$/].downcase; end
-end
-
 module Murlsh
 
   # URL ActiveRecord.
@@ -17,6 +13,10 @@ module Murlsh
       read_attribute(:title) || read_attribute(:url) || 'title missing'
     end
 
+    # Title with whitespace compressed and leading and trailing whitespace
+    # stripped.
+    def title_stripped; title.strip.gsub(/\s+/, ' '); end
+
     # Return true if this url has the same author as another url.
     def same_author?(other)
       other and other.email and other.name and

data/lib/murlsh/url_body.rb

@@ -54,15 +54,14 @@ module Murlsh
                   div(mu.name, :class => 'name') if mu.name
                 end
 
-                a(mu.title.strip.gsub(/\s+/, ' '), :href => mu.url)
+                a(mu.title_stripped, :href => mu.url)
 
                 mu.hostrec do |hostrec|
-                  text!(' ')
-                  span(hostrec, :class => 'host')
+                  self.sub(" [#{hostrec}]", :class => 'host')
                 end
                 mu.viarec do |via|
                   span(:class => 'via') {
-                    text!(' (via '); a(via.domain, :href => via); text!(')')
+                    text!(' via '); a(via.domain, :href => via)
                   }
                 end
                 span(", #{mu.time.fuzzy}", :class => 'date') if
@@ -115,14 +114,9 @@ module Murlsh
     # Search form builder.
     def search_form
       form(:action => '', :method => 'get') {
-        value = @q
-        Murlsh::Referrer.new(@req.referrer).search_query do |refq|
-          re_parts = refq.split.collect { |x| Regexp.escape(x) }
-          value = "\\b(#{re_parts.join('|')})\\b"
-        end
         fieldset {
           input(:type => 'text', :id => 'q', :name => 'q', :size => 32,
-            :value => value)
+            :value => @q)
           input(:type => 'submit', :value=> 'Regex Search')
         }
       }

data/lib/murlsh.rb

@@ -1,14 +1,15 @@
+require 'murlsh/doc'
+require 'murlsh/uri'
 require 'murlsh/auth'
 require 'murlsh/dispatch'
-require 'murlsh/get_content_type'
-require 'murlsh/get_title'
+require 'murlsh/failproof'
 require 'murlsh/openlock'
 require 'murlsh/plugin'
-require 'murlsh/referrer'
 require 'murlsh/sqlite3_adapter'
 require 'murlsh/time'
 require 'murlsh/url_server'
 require 'murlsh/url'
+require 'murlsh/uri_ask'
 require 'murlsh/xhtml_response'
 
 # requiring builder before active_record blows up

data/murlsh.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{murlsh}
-  s.version = "0.4.0"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matthew M. Boedicker"]
-  s.date = %q{2009-12-13}
+  s.date = %q{2010-01-15}
   s.default_executable = %q{murlsh}
   s.description = %q{url sharing site framework with easy adding, title lookup, atom feed, thumbnails and embedding}
   s.email = %q{matthewm@boedicker.org}
@@ -31,14 +31,15 @@ Gem::Specification.new do |s|
      "lib/murlsh/atom_feed.rb",
      "lib/murlsh/auth.rb",
      "lib/murlsh/dispatch.rb",
-     "lib/murlsh/get_content_type.rb",
-     "lib/murlsh/get_title.rb",
+     "lib/murlsh/doc.rb",
+     "lib/murlsh/failproof.rb",
      "lib/murlsh/markup.rb",
      "lib/murlsh/openlock.rb",
      "lib/murlsh/plugin.rb",
-     "lib/murlsh/referrer.rb",
      "lib/murlsh/sqlite3_adapter.rb",
      "lib/murlsh/time.rb",
+     "lib/murlsh/uri.rb",
+     "lib/murlsh/uri_ask.rb",
      "lib/murlsh/url.rb",
      "lib/murlsh/url_body.rb",
      "lib/murlsh/url_server.rb",
@@ -50,17 +51,15 @@ Gem::Specification.new do |s|
      "plugins/update_feed.rb",
      "public/css/jquery.jgrowl.css",
      "public/css/screen.css",
-     "public/js/jquery-1.3.2.min.js",
+     "public/js/jquery-1.4.min.js",
      "public/js/jquery.cookie.js",
      "public/js/jquery.jgrowl_compressed.js",
      "public/js/js.js",
      "public/swf/player_mp3_mini.swf",
+     "test/atom_feed_test.rb",
      "test/auth_test.rb",
-     "test/get_charset_test.rb",
-     "test/get_content_type_test.rb",
-     "test/get_title_test.rb",
      "test/markup_test.rb",
-     "test/referrer_test.rb",
+     "test/uri_ask_test.rb",
      "test/xhtml_response_test.rb"
   ]
   s.homepage = %q{http://github.com/mmb/murlsh}
@@ -70,12 +69,10 @@ Gem::Specification.new do |s|
   s.summary = %q{url sharing site framework}
   s.test_files = [
     "test/xhtml_response_test.rb",
+     "test/uri_ask_test.rb",
      "test/markup_test.rb",
-     "test/referrer_test.rb",
-     "test/get_charset_test.rb",
-     "test/get_content_type_test.rb",
-     "test/auth_test.rb",
-     "test/get_title_test.rb"
+     "test/atom_feed_test.rb",
+     "test/auth_test.rb"
   ]
 
   if s.respond_to? :specification_version then

data/plugins/lookup_content_type_title.rb

@@ -8,8 +8,9 @@ module Murlsh
     Hook = 'add_pre'
 
     def self.run(url, config)
-      url.content_type = Murlsh.get_content_type(url.url)
-      url.title = Murlsh.get_title(url.url, :content_type => url.content_type)
+      ask = URI(url.url).extend(Murlsh::UriAsk)
+      url.content_type = ask.content_type
+      url.title = ask.title
     end
 
   end

data/public/css/jquery.jgrowl.css

@@ -2,6 +2,8 @@
 div.jGrowl {
 	padding: 			10px;
 	z-index: 			9999;
+	color: 				#fff;
+	font-size: 			12px;
 }
 
 /** Special IE6 Style Positioning **/
@@ -76,16 +78,16 @@ div.center div.jGrowl-notification, div.center div.jGrowl-closer {
 
 div.jGrowl div.jGrowl-notification, div.jGrowl div.jGrowl-closer {
 	background-color: 		#000;
-	color: 					#fff;
 	opacity: 				.85;
-	filter: 				alpha(opacity = 85);
+    -ms-filter: 			"progid:DXImageTransform.Microsoft.Alpha(Opacity=85)"; 
+    filter: 				progid:DXImageTransform.Microsoft.Alpha(Opacity=85); 
 	zoom: 					1;
 	width: 					235px;
 	padding: 				10px;
 	margin-top: 			5px;
 	margin-bottom: 			5px;
 	font-family: 			Tahoma, Arial, Helvetica, sans-serif;
-	font-size: 				12px;
+	font-size: 				1em;
 	text-align: 			left;
 	display: 				none;
 	-moz-border-radius: 	5px;
@@ -98,23 +100,22 @@ div.jGrowl div.jGrowl-notification {
 
 div.jGrowl div.jGrowl-notification div.header {
 	font-weight: 			bold;
-	font-size:				10px;
+	font-size:				.85em;
 }
 
 div.jGrowl div.jGrowl-notification div.close {
 	z-index:				99;
 	float: 					right;
 	font-weight: 			bold;
-	font-size: 				12px;
+	font-size: 				1em;
 	cursor:					pointer;
 }
 
 div.jGrowl div.jGrowl-closer {
-	height: 				15px;
 	padding-top: 			4px;
 	padding-bottom: 		4px;
 	cursor: 				pointer;
-	font-size:				11px;
+	font-size:				.9em;
 	font-weight: 			bold;
 	text-align: 			center;
 }

data/public/css/screen.css

@@ -51,11 +51,19 @@ img.thumb, li object {
   margin-right : 10px;
 }
 
-span.host {
+sub.host {
   color : #808080;
   font-family : monospace;
 }
 
+span.via {
+  font-size : 0.75em;
+}
+
+span.via:before {
+  content : " \2190 \ ";
+}
+
 fieldset {
   border : 0;
   margin : 0;
@@ -121,7 +129,7 @@ div.jGrowl div.jGrowl-closer {
     width : 130px;
   }
 
-  input#url {
+  input#url, input#via {
     width : 160px;
   }