murlsh 1.3.1 → 1.4.0

data/lib/murlsh/uri_ask.rb

@@ -62,10 +62,7 @@ module Murlsh
       @description
     end
 
-    # Get the parsed doc at this url.
-    #
-    # Doc can be an Hpricot or Nokogiri doc or anything that supports the
-    # methods in Murlsh::Doc.
+    # Get the parsed Nokogiri doc at this url.
     #
     # Options:
     # * :failproof - if true hide all exceptions and return empty string on failure
@@ -78,14 +75,13 @@ module Murlsh
       if html?(options)
         Murlsh::failproof(options) do
           self.open(options[:headers]) do |f|
-            html_parse_plugins = Murlsh::Plugin.hooks('html_parse')
-            @doc = if html_parse_plugins.empty?
-              Nokogiri(f).extend(Murlsh::Doc)
-            else
-              html_parse_plugins.first.run(f).extend(Murlsh::Doc)
+            data = f.read
+            @doc = Nokogiri(data, to_s)
+            # encoding unknown, reparse with f.charset, default to utf-8
+            unless @doc.encoding
+              @doc = Nokogiri(data, to_s, f.charset || 'utf-8')
             end
-
-            @charset = @doc.charset || f.charset
+            @doc.extend(Murlsh::Doc)
           end
         end
       end
@@ -106,15 +102,15 @@ module Murlsh
       result
     end
 
-    HtmlContentTypePattern = %r{^text/html}i
+    HtmlContentTypeRe = %r{^text/html|application/xhtml\+xml}
 
     # Return true if the content type is HTML.
-    def html?(options={}); content_type(options)[HtmlContentTypePattern]; end
+    def html?(options={}); content_type(options)[HtmlContentTypeRe]; end
 
     # Convert from the character set of this url to utf-8 and decode HTML
     # entities.
     def decode(s)
-      HTMLEntities.new.decode(Iconv.conv('utf-8', @charset, s))
+      HTMLEntities.new.decode(Iconv.conv('utf-8', doc.encoding, s))
     end
 
     # Get the value of a response header.

data/lib/murlsh/url_body.rb

@@ -44,11 +44,9 @@ module Murlsh
         @body = html(:lang => 'en') {
           headd
           body {
+            self.p(:id => 'menu') { home_link ; text! ' | ' ; feed_link }
             search_form
-            self.p {
-              predefined_searches
-              feed_link
-            }
+            quick_search
             ul(:id => 'urls') {
               last = nil
 
@@ -105,21 +103,15 @@ module Murlsh
         css(@config['css_compressed'] || @config['css_files'])
         atom @config.fetch('feed_file')
         link :rel => 'first', :href => page_href(1)
-        link :rel => 'prev', :href => @prev_href  if @prev_href
-        link :rel => 'next', :href => @next_href  if @next_href
+        if p_href = prev_href
+          link :rel => 'prev', :href => p_href
+        end
+        if n_href = next_href
+          link :rel => 'next', :href => n_href
+        end
       }
     end
 
-    # Predefined search list builder.
-    def predefined_searches
-      if @config['predefined_searches']
-        text! 'search: '
-        @config['predefined_searches'].each do |k,v|
-          a "/#{k}", :href => "?q=#{URI.escape(v)}" ; text! ' '
-        end
-        text! '| '
-      end
-    end
 
     # Title builder.
     def titlee
@@ -127,9 +119,31 @@ module Murlsh
         (@req['q'] ? " /#{@req['q']}" : ''))
     end
 
+    # Home link builder.
+    def home_link; a 'Home', :href => @config.fetch('root_url'); end
+
     # Feed link builder.
     def feed_link
-      a('feed', :href => @config.fetch('feed_file'), :class => 'feed')
+      a 'Feed', :href => @config.fetch('feed_file'), :class => 'feed'
+    end
+
+    # Quick search list builder.
+    def quick_search
+      if @config['quick_search']
+        self.p {
+          text! 'Quick search: '
+          # can specify keys to be sorted first in quick_search_order config
+          # key, those keys will be first in given order, any keys not there
+          # will follow in natural sorted order
+          order = @config['quick_search_order'] || []
+          order += (@config['quick_search'].keys - order).sort
+          order.each do |k|
+            if v = @config['quick_search'][k]
+              a "/#{k}", :href => "?q=#{URI.escape(v)}" ; text! ' '
+            end
+          end
+        }
+      end
     end
 
     # Search form builder.
@@ -137,7 +151,7 @@ module Murlsh
       form(:action => '', :method => 'get') {
         fieldset {
           form_input :id => 'q', :size => 32, :value => @req['q']
-          form_input :type => 'submit', :value => 'Regex Search'
+          form_input :type => 'submit', :value => 'Search'
         }
       }
     end
@@ -176,7 +190,7 @@ module Murlsh
     # Powered by builder.
     def powered_by
       self.p {
-        text! 'powered by '
+        text! 'Powered by '
         a 'murlsh', :href => 'http://github.com/mmb/murlsh/'
       }
     end

data/lib/murlsh/url_result_set.rb

@@ -22,9 +22,19 @@ module Murlsh
         :limit => per_page, :offset => offset)
     end
 
-    def prev_page; @prev_page ||= page - 1  if (2..total_pages) === page; end
+    def prev_page
+      unless instance_variable_defined? :@prev_page
+        @prev_page = page - 1  if (2..total_pages) === page
+      end
+      @prev_page
+    end
 
-    def next_page; @next_page ||= page + 1  if page < total_pages; end
+    def next_page
+      unless instance_variable_defined? :@next_page
+        @next_page = page + 1  if page < total_pages
+      end
+      @next_page
+    end
 
     attr_reader :conditions
     attr_reader :page

data/lib/murlsh.rb

@@ -1,9 +1,11 @@
 require 'murlsh/head_from_get'
 require 'murlsh/search_conditions'
 
+require 'murlsh/ask'
 require 'murlsh/auth'
 require 'murlsh/build_md5'
 require 'murlsh/build_query'
+require 'murlsh/cp_r_safe'
 require 'murlsh/delicious_parse'
 require 'murlsh/dispatch'
 require 'murlsh/doc'
@@ -12,6 +14,7 @@ require 'murlsh/failproof'
 require 'murlsh/far_future_expires'
 require 'murlsh/image_list'
 require 'murlsh/img_store'
+require 'murlsh/install'
 require 'murlsh/json_body'
 require 'murlsh/json_server'
 require 'murlsh/jsonp_body'
@@ -19,7 +22,7 @@ require 'murlsh/markup'
 require 'murlsh/must_revalidate'
 require 'murlsh/openlock'
 require 'murlsh/plugin'
-require 'murlsh/sqlite3_adapter'
+require 'murlsh/search_grammar'
 require 'murlsh/time_ago'
 require 'murlsh/uri_ask'
 require 'murlsh/uri_domain'

data/murlsh.gemspec

@@ -5,13 +5,13 @@
 
 Gem::Specification.new do |s|
   s.name = %q{murlsh}
-  s.version = "1.3.1"
+  s.version = "1.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matthew M. Boedicker"]
-  s.date = %q{2011-01-15}
+  s.date = %q{2011-01-31}
   s.default_executable = %q{murlsh}
-  s.description = %q{url sharing site framework with easy adding, title lookup, atom feed, thumbnails and embedding}
+  s.description = %q{Host your bookmarks or maintain a link blog}
   s.email = %q{matthewm@boedicker.org}
   s.executables = ["murlsh"]
   s.extra_rdoc_files = [
@@ -27,9 +27,11 @@ Gem::Specification.new do |s|
     "config.ru",
     "config.yaml",
     "lib/murlsh.rb",
+    "lib/murlsh/ask.rb",
     "lib/murlsh/auth.rb",
     "lib/murlsh/build_md5.rb",
     "lib/murlsh/build_query.rb",
+    "lib/murlsh/cp_r_safe.rb",
     "lib/murlsh/delicious_parse.rb",
     "lib/murlsh/dispatch.rb",
     "lib/murlsh/doc.rb",
@@ -39,6 +41,7 @@ Gem::Specification.new do |s|
     "lib/murlsh/head_from_get.rb",
     "lib/murlsh/image_list.rb",
     "lib/murlsh/img_store.rb",
+    "lib/murlsh/install.rb",
     "lib/murlsh/json_body.rb",
     "lib/murlsh/json_server.rb",
     "lib/murlsh/jsonp_body.rb",
@@ -47,7 +50,8 @@ Gem::Specification.new do |s|
     "lib/murlsh/openlock.rb",
     "lib/murlsh/plugin.rb",
     "lib/murlsh/search_conditions.rb",
-    "lib/murlsh/sqlite3_adapter.rb",
+    "lib/murlsh/search_grammar.rb",
+    "lib/murlsh/search_grammar.treetop",
     "lib/murlsh/time_ago.rb",
     "lib/murlsh/uri_ask.rb",
     "lib/murlsh/uri_domain.rb",
@@ -59,26 +63,25 @@ Gem::Specification.new do |s|
     "lib/murlsh/yaml_ordered_hash.rb",
     "murlsh.gemspec",
     "plugins/add_post_50_update_feed.rb",
+    "plugins/add_post_50_update_m3u.rb",
     "plugins/add_post_50_update_podcast.rb",
     "plugins/add_post_50_update_rss.rb",
     "plugins/add_post_60_notify_hubs.rb",
+    "plugins/add_pre_30_unajax_twitter.rb",
+    "plugins/add_pre_35_url_clean.rb",
     "plugins/add_pre_40_convert_mobile.rb",
-    "plugins/add_pre_41_unajax_twitter.rb",
     "plugins/add_pre_45_supplied_thumbnail.rb",
     "plugins/add_pre_50_lookup_content_type_title.rb",
     "plugins/add_pre_50_media_thumbnail.rb",
     "plugins/add_pre_50_open_graph_image.rb",
-    "plugins/add_pre_60_flickr.rb",
     "plugins/add_pre_60_github_title.rb",
     "plugins/add_pre_60_google_code_title.rb",
     "plugins/add_pre_60_imgur.rb",
     "plugins/add_pre_60_s3_image.rb",
     "plugins/add_pre_60_twitter.rb",
-    "plugins/add_pre_60_vimeo.rb",
     "plugins/add_pre_65_html_thumb.rb",
     "plugins/add_pre_65_img_thumb.rb",
     "plugins/avatar_50_gravatar.rb",
-    "plugins/html_parse_50_nokogiri.rb",
     "plugins/url_display_add_45_audio.rb",
     "plugins/url_display_add_50_hostrec.rb",
     "plugins/url_display_add_55_content_type.rb",
@@ -87,11 +90,10 @@ Gem::Specification.new do |s|
     "public/css/jquery.jgrowl.css",
     "public/css/screen.css",
     "public/img/thumb/.gitignore",
-    "public/js/comments.json",
-    "public/js/jquery-1.4.4.min.js",
+    "public/js/jquery-1.5.min.js",
     "public/js/jquery.jgrowl_compressed.js",
     "public/js/js.js",
-    "public/js/twitter-text-1.0.4.js",
+    "public/js/twitter-text-1.3.1.js",
     "spec/auth_spec.rb",
     "spec/dispatch_spec.rb",
     "spec/doc_spec.rb",
@@ -105,7 +107,7 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/mmb/murlsh}
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.4.2}
-  s.summary = %q{url sharing site framework}
+  s.summary = %q{Host your bookmarks or maintain a link blog}
   s.test_files = [
     "spec/auth_spec.rb",
     "spec/dispatch_spec.rb",
@@ -125,11 +127,11 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_runtime_dependency(%q<bcrypt-ruby>, [">= 2.1.2"])
       s.add_runtime_dependency(%q<builder>, [">= 2.1.2"])
-      s.add_runtime_dependency(%q<flickraw>, [">= 0.8.3"])
       s.add_runtime_dependency(%q<htmlentities>, [">= 4.2.0"])
       s.add_runtime_dependency(%q<json>, [">= 1.2.3"])
       s.add_runtime_dependency(%q<nokogiri>, ["~> 1.0"])
       s.add_runtime_dependency(%q<plumnailer>, [">= 0.1.0"])
+      s.add_runtime_dependency(%q<postrank-uri>, ["~> 1.0"])
       s.add_runtime_dependency(%q<public_suffix_service>, ["~> 0.0"])
       s.add_runtime_dependency(%q<push-notify>, [">= 0.1.0"])
       s.add_runtime_dependency(%q<rack>, [">= 1.0.0"])
@@ -137,10 +139,11 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<rack-rewrite>, [">= 1.0.2"])
       s.add_runtime_dependency(%q<rack-throttle>, [">= 0.3.0"])
       s.add_runtime_dependency(%q<rmagick>, [">= 1.15.14"])
-      s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 1.2.1"])
+      s.add_runtime_dependency(%q<sqlite3>, ["~> 1.3"])
       s.add_runtime_dependency(%q<tinyatom>, [">= 0.3.3"])
+      s.add_runtime_dependency(%q<treetop>, ["~> 1.4"])
       s.add_runtime_dependency(%q<twitter>, [">= 0.9.12"])
-      s.add_runtime_dependency(%q<vimeo>, [">= 1.2.2"])
+      s.add_development_dependency(%q<fakeweb>, ["~> 1.3"])
       s.add_development_dependency(%q<flog>, [">= 2.5.0"])
       s.add_development_dependency(%q<rack-test>, ["~> 0.5"])
       s.add_development_dependency(%q<rspec>, ["~> 2.0"])
@@ -148,11 +151,11 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<activerecord>, [">= 2.3.4"])
       s.add_dependency(%q<bcrypt-ruby>, [">= 2.1.2"])
       s.add_dependency(%q<builder>, [">= 2.1.2"])
-      s.add_dependency(%q<flickraw>, [">= 0.8.3"])
       s.add_dependency(%q<htmlentities>, [">= 4.2.0"])
       s.add_dependency(%q<json>, [">= 1.2.3"])
       s.add_dependency(%q<nokogiri>, ["~> 1.0"])
       s.add_dependency(%q<plumnailer>, [">= 0.1.0"])
+      s.add_dependency(%q<postrank-uri>, ["~> 1.0"])
       s.add_dependency(%q<public_suffix_service>, ["~> 0.0"])
       s.add_dependency(%q<push-notify>, [">= 0.1.0"])
       s.add_dependency(%q<rack>, [">= 1.0.0"])
@@ -160,10 +163,11 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rack-rewrite>, [">= 1.0.2"])
       s.add_dependency(%q<rack-throttle>, [">= 0.3.0"])
       s.add_dependency(%q<rmagick>, [">= 1.15.14"])
-      s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.1"])
+      s.add_dependency(%q<sqlite3>, ["~> 1.3"])
       s.add_dependency(%q<tinyatom>, [">= 0.3.3"])
+      s.add_dependency(%q<treetop>, ["~> 1.4"])
       s.add_dependency(%q<twitter>, [">= 0.9.12"])
-      s.add_dependency(%q<vimeo>, [">= 1.2.2"])
+      s.add_dependency(%q<fakeweb>, ["~> 1.3"])
       s.add_dependency(%q<flog>, [">= 2.5.0"])
       s.add_dependency(%q<rack-test>, ["~> 0.5"])
       s.add_dependency(%q<rspec>, ["~> 2.0"])
@@ -172,11 +176,11 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<activerecord>, [">= 2.3.4"])
     s.add_dependency(%q<bcrypt-ruby>, [">= 2.1.2"])
     s.add_dependency(%q<builder>, [">= 2.1.2"])
-    s.add_dependency(%q<flickraw>, [">= 0.8.3"])
     s.add_dependency(%q<htmlentities>, [">= 4.2.0"])
     s.add_dependency(%q<json>, [">= 1.2.3"])
     s.add_dependency(%q<nokogiri>, ["~> 1.0"])
     s.add_dependency(%q<plumnailer>, [">= 0.1.0"])
+    s.add_dependency(%q<postrank-uri>, ["~> 1.0"])
     s.add_dependency(%q<public_suffix_service>, ["~> 0.0"])
     s.add_dependency(%q<push-notify>, [">= 0.1.0"])
     s.add_dependency(%q<rack>, [">= 1.0.0"])
@@ -184,10 +188,11 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<rack-rewrite>, [">= 1.0.2"])
     s.add_dependency(%q<rack-throttle>, [">= 0.3.0"])
     s.add_dependency(%q<rmagick>, [">= 1.15.14"])
-    s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.1"])
+    s.add_dependency(%q<sqlite3>, ["~> 1.3"])
     s.add_dependency(%q<tinyatom>, [">= 0.3.3"])
+    s.add_dependency(%q<treetop>, ["~> 1.4"])
     s.add_dependency(%q<twitter>, [">= 0.9.12"])
-    s.add_dependency(%q<vimeo>, [">= 1.2.2"])
+    s.add_dependency(%q<fakeweb>, ["~> 1.3"])
     s.add_dependency(%q<flog>, [">= 2.5.0"])
     s.add_dependency(%q<rack-test>, ["~> 0.5"])
     s.add_dependency(%q<rspec>, ["~> 2.0"])

data/plugins/add_post_50_update_m3u.rb

@@ -0,0 +1,35 @@
+require 'murlsh'
+
+module Murlsh
+
+  # Regenerate m3u file after a new audio url has been added.
+  class AddPost50UpdateM3u < Plugin
+
+    @hook = 'add_post'
+
+    AudioContentTypes = %w{
+      application/ogg
+      audio/mpeg
+      audio/ogg
+      }
+
+    OutputFile = 'm3u.m3u'
+
+    def self.run(url, config)
+      if AudioContentTypes.include?(url.content_type) or
+        not File.exists?(OutputFile)
+
+        Murlsh::openlock(OutputFile, 'w') do |f|
+          f.write "# #{config['root_url']}\r\n\r\n"
+          Murlsh::Url.all(:conditions =>
+            ["content_type IN (?)", AudioContentTypes],
+            :order => 'time DESC').each do |mu|
+            f.write "#{mu.url}\r\n"
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/plugins/{add_pre_41_unajax_twitter.rb → add_pre_30_unajax_twitter.rb}

@@ -3,7 +3,7 @@ require 'murlsh'
 module Murlsh
 
   # Convert Ajax friendly Twitter urls (with #!) into usable urls.
-  class AddPre41UnajaxTwitter < Plugin
+  class AddPre30UnajaxTwitter < Plugin
 
     @hook = 'add_pre'
 

data/plugins/add_pre_35_url_clean.rb

@@ -0,0 +1,21 @@
+require 'postrank-uri'
+
+require 'murlsh'
+
+module Murlsh
+
+  # Canonicalize and clean urls with postrank-uri.
+  #
+  # See https://github.com/postrank-labs/postrank-uri
+  class AddPre35UrlClean < Plugin
+
+    @hook = 'add_pre'
+
+    def self.run(url, config)
+      url.url = PostRank::URI.clean(url.url)
+      url.via = PostRank::URI.clean(url.via)  if url.via
+    end
+
+  end
+
+end

data/plugins/add_pre_50_media_thumbnail.rb

@@ -4,7 +4,7 @@ require 'murlsh'
 
 module Murlsh
 
-  # If document has <meta rel="media:thumbnail"> use it as the thumbnail.
+  # If document has meta or link media:thumbnail use it as the thumbnail.
   class AddPre50MediaThumbnail < Plugin
 
     @hook = 'add_pre'
@@ -14,8 +14,11 @@ module Murlsh
 
     def self.run(url, config)
       if not url.thumbnail_url and url.ask.doc
-        url.ask.doc.xpath_search("//meta[@rel='media:thumbnail']") do |node|
-          if node and node['href'] and not node['href'].empty?
+        url.ask.doc.xpath_search(%w{
+          //meta[@rel='media:thumbnail']
+          //link[@rel='media:thumbnail']
+          }) do |node|
+          if node and not node['href'].to_s.empty?
             Murlsh::failproof do
               thumb_storage = Murlsh::ImgStore.new(StorageDir,
                 :user_agent => config['user_agent'])

data/plugins/add_pre_65_html_thumb.rb

@@ -14,11 +14,9 @@ module Murlsh
     StorageDir = File.join(File.dirname(__FILE__), '..', 'public', 'img',
       'thumb')
 
-    MarkupContentTypeRe = %r{^text/html|application/xhtml\+xml}
-
     def self.run(url, config)
       if not url.thumbnail_url and url.content_type and
-        url.content_type[MarkupContentTypeRe]
+        url.content_type[Murlsh::UriAsk::HtmlContentTypeRe]
         Murlsh::failproof do
           chooser = Plumnailer::Chooser.new
           choice = chooser.choose(url.url)

data/plugins/url_display_add_60_via.rb

@@ -9,12 +9,13 @@ module Murlsh
 
     @hook = 'url_display_add'
 
-    HttpRe = %r{^http://}i
+    HttpRe = %r{^https?://}i
     HackerNewsRe = %r{^news\.ycombinator\.com}i
     RedditRe = %r{^www\.reddit\.com/r/([a-z\d]+?)/}i
     DeliciousRe = %r{^(?:www\.)?delicious\.com/(\w+)}i
     TwitterRe = %r{^twitter\.com/(\w+)}i
     TumblrRe = %r{^([a-z\d][a-z\d-]{0,61}[a-z\d])\.tumblr\.com/}i
+    PinboardRe = %r{^pinboard\.in/(popular|[tu]:[^/]+(?:/t:[^/]+)?)/?$}i
 
     # Show a via link for the url.
     def self.run(markup, url, config)
@@ -29,6 +30,7 @@ module Murlsh
             when m = search.match(DeliciousRe); "delicious/#{m[1]}"
             when m = search.match(TwitterRe); "twitter/#{m[1]}"
             when m = search.match(TumblrRe); "#{m[1]}.tumblr"
+            when m = search.match(PinboardRe); "pinboard/#{m[1]}"
             else via_uri.extend(Murlsh::URIDomain).domain || via_uri_s
           end
 
@@ -43,3 +45,4 @@ module Murlsh
   end
 
 end
+

data/public/css/jquery.jgrowl.css

@@ -40,31 +40,35 @@ div.ie6.center {
 }
 
 /** Normal Style Positions **/
+div.jGrowl {
+	position:			absolute;
+}
+
 body > div.jGrowl {
 	position:			fixed;
 }
 
-body > div.jGrowl.top-left {
+div.jGrowl.top-left {
 	left: 				0px;
 	top: 				0px;
 }
 
-body > div.jGrowl.top-right {
+div.jGrowl.top-right {
 	right: 				0px;
 	top: 				0px;
 }
 
-body > div.jGrowl.bottom-left {
+div.jGrowl.bottom-left {
 	left: 				0px;
 	bottom:				0px;
 }
 
-body > div.jGrowl.bottom-right {
+div.jGrowl.bottom-right {
 	right: 				0px;
 	bottom: 			0px;
 }
 
-body > div.jGrowl.center {
+div.jGrowl.center {
 	top: 				0px;
 	width: 				50%;
 	left: 				25%;
@@ -95,12 +99,12 @@ div.jGrowl div.jGrowl-notification {
 	min-height: 			40px;
 }
 
-div.jGrowl div.jGrowl-notification div.header {
+div.jGrowl div.jGrowl-notification div.jGrowl-header {
 	font-weight: 			bold;
 	font-size:				.85em;
 }
 
-div.jGrowl div.jGrowl-notification div.close {
+div.jGrowl div.jGrowl-notification div.jGrowl-close {
 	z-index:				99;
 	float: 					right;
 	font-weight: 			bold;

data/public/css/screen.css

@@ -1,4 +1,5 @@
 body {
+  color : #333;
   font-size : 1em;
   line-height : 1.5em;
 }
@@ -9,7 +10,7 @@ img {
 
 #urls {
   list-style-type : none;
-  width : 600px;
+  width : 50%;
 }
 
 ul.comments {
@@ -29,7 +30,7 @@ div.icon {
 }
 
 a:link {
-  color : #000;
+  color : #333;
 }
 
 a:visited {