murlsh 1.2.0 → 1.2.1

data/README.textile

@@ -11,6 +11,7 @@ Site for sharing and archiving links.
 * plugin interface
 * rack interface
 * Gravatar support
+* imports Netscape bookmark format files
 
 See "http://urls.matthewm.boedicker.org/":http://urls.matthewm.boedicker.org/ for example.
 

data/Rakefile

@@ -4,9 +4,11 @@ require 'cgi'
 require 'digest/md5'
 require 'net/http'
 require 'pp'
+require 'set'
 require 'uri'
 require 'yaml'
 
+require 'active_record'
 require 'RMagick'
 require 'sqlite3'
 
@@ -70,7 +72,7 @@ namespace :db do
   task :init do
     puts "creating #{config.fetch('db_file')}"
     db = SQLite3::Database.new(config.fetch('db_file'))
-    db.execute('CREATE TABLE urls (
+    db.execute 'CREATE TABLE urls (
       id INTEGER PRIMARY KEY,
       time TIMESTAMP,
       url TEXT,
@@ -81,7 +83,8 @@ namespace :db do
       content_type TEXT,
       via TEXT,
       thumbnail_url TEXT);
-      ')
+      '
+    db.execute 'CREATE INDEX IF NOT EXISTS urls_time_desc ON urls (time DESC);'
   end
 
   desc 'Interact with the database.'
@@ -133,13 +136,6 @@ begin
   Spec::Rake::SpecTask.new('test') do |t|
     t.spec_files = FileList['spec/*_spec.rb']
     t.spec_opts = %w{--color}
-    # list of places to check for unicode_formatter.rb and use it if found
-    %w{unicode_formatter.rb}.map { |x| File.expand_path(x) }.each do |f|
-      if File.exists?(f)
-        t.spec_opts.push(*%W{--require #{f} --format UnicodeFormatter})
-        break
-      end
-    end
     t.verbose = true
     t.warning = true
   end
@@ -183,7 +179,7 @@ namespace :user do
 end
 
 # Validate a document with the W3C validation service.
-def validate(check_url, options={})
+def validate_html(check_url, options={})
   opts = {
     :validator_host => 'validator.w3.org',
     :validator_port => 80,
@@ -217,7 +213,7 @@ namespace :validate do
   task :html do
     check_url = config['root_url']
     print "validating #{check_url} : "
-    result = validate(check_url)
+    result = validate_html(check_url)
     if Net::HTTPSuccess === result[:response]
       puts "#{result[:status]} (#{result[:errors]} errors, #{result[:warnings]} warnings)"
     else
@@ -346,11 +342,13 @@ namespace :thumb do
   task :check do
     ActiveRecord::Base.establish_connection :adapter => 'sqlite3',
       :database => config.fetch('db_file')
+    used_thumbnails = Set.new
     Murlsh::Url.all(
       :conditions => "thumbnail_url like 'img/thumb/%'").each do |u|
       identity = "url #{u.id} (#{u.url})"
 
       path = File.join(%w{public}.concat(File.split(u.thumbnail_url)))
+      used_thumbnails.add(path)
       if File.readable?(path)
         img_data = open(path) { |f| f.read }
 
@@ -376,6 +374,40 @@ namespace :thumb do
         puts "#{identity} thumbnail #{path} does not exist or is not readable"
       end
     end
+    # check if all thumbnail files that exist are in the database
+    (Dir['public/img/thumb/*'] - used_thumbnails.to_a).each do |t|
+      puts "thumbnail #{t} is not used"
+    end
+  end
+
+end
+
+namespace :import do
+
+  desc 'Convert a delicious xml export into an import shell script.'
+  task :delicious, :source do |t, args|
+    puts <<EOS
+#!/bin/sh
+
+# murlsh import, source #{args.source}
+
+PASSWORD="$1"
+if [ -z "${PASSWORD}" ] ; then
+    echo 'Password not set, pass as command line argument or hardcode in script'
+    exit 1
+fi
+
+SITE_URL='#{config.fetch('root_url')}'
+
+EOS
+    Murlsh.delicious_parse(args.source) do |b|
+      # escape single quotes because these will be in single quotes in output
+      href_escaped = b[:href].to_s.gsub("'", "'\"'\"'")
+      via_url_escaped = b[:via_url].to_s.gsub("'", "'\"'\"'")
+      puts <<EOS
+curl --data-urlencode 'url=#{href_escaped}' --data-urlencode "auth=${PASSWORD}" --data-urlencode 'via=#{via_url_escaped}' --data-urlencode 'time=#{b[:time].to_i}' ${SITE_URL}
+EOS
+    end
   end
 
 end
@@ -422,6 +454,7 @@ begin
       }.each_slice(3) { |g,o,v| gemspec.add_dependency(g, "#{o} #{v}") }
     %w{
       flog >= 2.5.0
+      rack-test ~> 0.5
       rspec ~> 1.3
       }.each_slice(3) do |g,o,v|
       gemspec.add_development_dependency(g, "#{o} #{v}")

data/VERSION

@@ -1 +1 @@
-1.2.0
+1.2.1

data/config.ru

@@ -41,6 +41,6 @@ end
 
 # use Rack::Lint
 
-Dir['plugins/*.rb'].each { |p| require p }
+Dir['plugins/*.rb'].each { |p| require "./#{p}" }
 
 run Murlsh::Dispatch.new(config)

data/config.yaml

@@ -20,7 +20,7 @@ meta_tag_description: URLs found interesting by Matthew M. Boedicker
 meta_tag_verify-v1: 
 meta_tag_viewport: width=device-width,minimum-scale=1.0,maximum-scale=1.0
 num_posts_feed: 25
-num_posts_page: 100
+num_posts_page: 25
 page_title: mmb url share
 pubsubhubbub_hubs: []
 

data/lib/murlsh/auth.rb

@@ -17,9 +17,20 @@ module Murlsh
 
     def initialize(file); @file = file; end
 
+    # Handle differences in csv interface between ruby 1.8 and 1.9.
+    def self.csv_iter(csv_file, &block)
+      if defined?(CSV::Reader)
+        # ruby 1.8
+        CSV::Reader.parse(open(csv_file), &block)
+      else
+        # ruby 1.9
+        CSV.foreach(csv_file, &block)
+      end
+    end
+
     # Authenticate a user by password. Return their name and email if correct.
     def auth(password)
-      CSV::Reader.parse(open(@file)) do |row|
+      self.class.csv_iter(@file) do |row|
         return { :name => row[0], :email => row[1] }  if
           BCrypt::Password.new(row[2]) == password
       end

data/lib/murlsh/delicious_parse.rb

@@ -0,0 +1,38 @@
+require 'open-uri'
+require 'uri'
+
+require 'nokogiri'
+
+module Murlsh
+
+  module_function
+
+  # Parse a delicious xml export and yield a hash for each bookmark.
+  #
+  # To export your delicious bookmarks:
+  #   curl https://user:password@api.del.icio.us/v1/posts/all > delicious.xml
+  def delicious_parse(source)
+    doc = Nokogiri::XML(open(source))
+
+    doc.xpath('//post').each do |p|
+      result = {}
+      p.each { |k,v| result[k.to_sym] = v }
+
+      result[:tag] = result[:tag].split
+      result[:time] = Time.parse(result[:time])
+
+      # extract via information from extended
+      result[:via] = result[:extended].chomp(')')[%r{via\s+([^\s]+)}, 1]
+      result[:via_url] = begin
+        if result[:via] and
+          %w{http https}.include?(URI(result[:via]).scheme.to_s.downcase)
+          result[:via]
+        end
+      rescue URI::InvalidURIError
+      end
+
+      yield result
+    end
+  end
+
+end

data/lib/murlsh/dispatch.rb

@@ -35,6 +35,7 @@ module Murlsh
 
       ActiveRecord::Base.default_timezone = :utc
       ActiveRecord::Base.include_root_in_json = false
+      # ActiveRecord::Base.logger = Logger.new(STDERR)
     end
 
     # Figure out which method will handle request.

data/lib/murlsh/doc.rb

@@ -10,7 +10,7 @@ module Murlsh
         unless content_type.nil?
           content = content_type['content']
           unless content.nil?
-            charset = content[/charset=([\w_.:-]+)/, 1]
+            charset = content[/charset=([\w.:-]+)/, 1]
             return charset  if charset
           end
         end

data/lib/murlsh/uri_ask.rb

@@ -99,7 +99,7 @@ module Murlsh
         'User-Agent' =>
           'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
         }
-      if (host || '')[/^www\.nytimes\.com/]
+      if host.to_s[/^www\.nytimes\.com/]
         result['Referer'] = 'http://news.google.com/'
       end
 
@@ -125,7 +125,7 @@ module Murlsh
     def header(header_name, options={})
       result = [*head_headers(options)[header_name]][0]
       result = get_headers(options)[header_name]  if not result or result.empty?
-      result || ''
+      result.to_s
     end
 
     # Get and cache response headers returned by HTTP HEAD for this URI.

data/lib/murlsh/url_body.rb

@@ -9,14 +9,44 @@ module Murlsh
     def initialize(config, req, content_type='text/html')
       @config, @req, @q, @content_type =
         config, req, req.params['q'], content_type
+      @page = [req.params['p'].to_i, 1].max
+
+      @first_href, @prev_href, @next_href = page_href(1), nil, nil
+ 
+      @total_entries, @total_pages = 0, 0
+
+      @per_page = @req.params['pp'] ? @req.params['pp'].to_i :
+        config.fetch('num_posts_page', 25)
+
       super(:indent => @config['html_indent'] || 0)
     end
 
+    # Get the href of a page in the same result set as this page.
+    def page_href(page)
+      query = @req.params.dup
+      query['p'] = page
+      Murlsh.build_query(query)
+    end
+
     # Fetch urls based on query string parameters.
     def urls
-      Murlsh::Url.all(:conditions => search_conditions, :order => 'id DESC',
-        :limit =>  @req.params['n'] ? @req.params['n'].to_i :
-          @config.fetch('num_posts_page', 100))
+      search = search_conditions
+
+      @total_entries = Murlsh::Url.count(:conditions => search)
+      @total_pages = (@total_entries / @per_page.to_f).ceil
+
+      if @page > 1 and @page <= @total_pages
+        @prev_href = page_href(@page - 1)
+      end
+
+      if @page < @total_pages
+        @next_href = page_href(@page + 1)
+      end
+
+      offset = (@page - 1) * @per_page
+
+      Murlsh::Url.all(:conditions => search_conditions, :order => 'time DESC',
+        :limit => @per_page, :offset => offset)
     end
 
     # Search conditions builder for ActiveRecord conditions.
@@ -32,6 +62,8 @@ module Murlsh
 
     # Url list page body builder.
     def each
+      mus = urls
+
       declare! :DOCTYPE, :html
 
       yield html(:lang => 'en') {
@@ -41,7 +73,8 @@ module Murlsh
             li { feed_icon ; search_form }
 
             last = nil
-            urls.each do |mu|
+
+            mus.each do |mu|
               li {
                 unless mu.same_author?(last)
                   avatar_url = Murlsh::Plugin.hooks('avatar').inject(
@@ -70,6 +103,8 @@ module Murlsh
               }
             end
 
+            li { paging_nav }
+
             li { add_form }
           }
 
@@ -90,6 +125,9 @@ module Murlsh
           map { |k,v| [k.sub('meta_tag_', ''), v] })
         css(@config['css_compressed'] || @config['css_files'])
         atom @config.fetch('feed_file')
+        link :rel => 'first', :href => @first_href
+        link :rel => 'prev', :href => @prev_href  if @prev_href
+        link :rel => 'next', :href => @next_href  if @next_href
       }
     end
 
@@ -115,6 +153,17 @@ module Murlsh
       }
     end
 
+    # Paging navigation.
+    def paging_nav
+      text! "Page #{@page}/#{@total_pages}"
+      if @prev_href
+        text! ' | '; a 'previous', :href => @prev_href
+      end
+      if @next_href
+        text! ' | '; a 'next', :href => @next_href
+      end
+    end
+
     # Url add form builder.
     def add_form
       form(:action => '', :method => 'post') {

data/lib/murlsh/url_server.rb

@@ -36,11 +36,15 @@ module Murlsh
         @config.fetch('auth_file')).auth(auth)
 
         mu = Murlsh::Url.new do |u|
-          u.time = Time.now.gmtime
+          u.time = if req.params['time']
+            Time.at(req.params['time'].to_f).utc
+          else
+            Time.now.utc
+          end
           u.url = req.params['url']
           u.email = user[:email]
           u.name = user[:name]
-          u.via = req.params['via']  unless (req.params['via'] || []).empty?
+          u.via = req.params['via']  unless req.params['via'].to_s.empty?
         end
 
         begin

data/lib/murlsh.rb

@@ -3,6 +3,7 @@ require 'murlsh/head_from_get'
 require 'murlsh/auth'
 require 'murlsh/build_query'
 require 'murlsh/config_server'
+require 'murlsh/delicious_parse'
 require 'murlsh/dispatch'
 require 'murlsh/doc'
 require 'murlsh/etag_add_encoding'

data/murlsh.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{murlsh}
-  s.version = "1.2.0"
+  s.version = "1.2.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matthew M. Boedicker"]
-  s.date = %q{2010-12-16}
+  s.date = %q{2010-12-23}
   s.default_executable = %q{murlsh}
   s.description = %q{url sharing site framework with easy adding, title lookup, atom feed, thumbnails and embedding}
   s.email = %q{matthewm@boedicker.org}
@@ -30,6 +30,7 @@ Gem::Specification.new do |s|
     "lib/murlsh/auth.rb",
     "lib/murlsh/build_query.rb",
     "lib/murlsh/config_server.rb",
+    "lib/murlsh/delicious_parse.rb",
     "lib/murlsh/dispatch.rb",
     "lib/murlsh/doc.rb",
     "lib/murlsh/etag_add_encoding.rb",
@@ -57,6 +58,7 @@ Gem::Specification.new do |s|
     "plugins/add_post_50_update_rss.rb",
     "plugins/add_post_60_notify_hubs.rb",
     "plugins/add_pre_40_convert_mobile.rb",
+    "plugins/add_pre_41_unajax_twitter.rb",
     "plugins/add_pre_50_lookup_content_type_title.rb",
     "plugins/add_pre_50_media_thumbnail.rb",
     "plugins/add_pre_50_open_graph_image.rb",
@@ -135,6 +137,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<twitter>, [">= 0.9.12"])
       s.add_runtime_dependency(%q<vimeo>, [">= 1.2.2"])
       s.add_development_dependency(%q<flog>, [">= 2.5.0"])
+      s.add_development_dependency(%q<rack-test>, ["~> 0.5"])
       s.add_development_dependency(%q<rspec>, ["~> 1.3"])
     else
       s.add_dependency(%q<activerecord>, [">= 2.3.4"])
@@ -157,6 +160,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<twitter>, [">= 0.9.12"])
       s.add_dependency(%q<vimeo>, [">= 1.2.2"])
       s.add_dependency(%q<flog>, [">= 2.5.0"])
+      s.add_dependency(%q<rack-test>, ["~> 0.5"])
       s.add_dependency(%q<rspec>, ["~> 1.3"])
     end
   else
@@ -180,6 +184,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<twitter>, [">= 0.9.12"])
     s.add_dependency(%q<vimeo>, [">= 1.2.2"])
     s.add_dependency(%q<flog>, [">= 2.5.0"])
+    s.add_dependency(%q<rack-test>, ["~> 0.5"])
     s.add_dependency(%q<rspec>, ["~> 1.3"])
   end
 end

data/plugins/add_post_50_update_feed.rb

@@ -26,7 +26,7 @@ module Murlsh
         :hubs => config.fetch('pubsubhubbub_hubs', []).
           map { |x| x['subscribe_url'] } )
 
-      latest = Murlsh::Url.all(:order => 'id DESC',
+      latest = Murlsh::Url.all(:order => 'time DESC',
         :limit => config.fetch('num_posts_feed', 25))
 
       latest.each do |mu|

data/plugins/add_post_50_update_podcast.rb

@@ -21,7 +21,7 @@ module Murlsh
           f.items.do_sort = true
   
           Murlsh::Url.all(:conditions => { :content_type => 'audio/mpeg' },
-            :order => 'id DESC',
+            :order => 'time DESC',
             :limit => config.fetch('num_posts_feed', 25)).each do |mu|
             i = f.items.new_item
             i.title = mu.title_stripped

data/plugins/add_post_50_update_rss.rb

@@ -27,7 +27,7 @@ module Murlsh
         f.channel.link = URI.join(config.fetch('root_url'), output_file)
         f.items.do_sort = true
   
-        Murlsh::Url.all(:order => 'id DESC',
+        Murlsh::Url.all(:order => 'time DESC',
           :limit => config.fetch('num_posts_feed', 25)).each do |mu|
           i = f.items.new_item
           i.title = mu.title_stripped

data/plugins/add_pre_40_convert_mobile.rb

@@ -11,14 +11,19 @@ module Murlsh
     TwitterRe = %r{^(http://)mobile\.(twitter\.com/.*)$}i
     WikipediaRe = %r{^(http://[a-z]+\.)m\.(wikipedia\.org/.*)$}i
 
-    def self.run(url, config)
-      url.url = case
-        when match = TwitterRe.match(url.url); "#{match[1]}#{match[2]}"
-        when match = WikipediaRe.match(url.url); "#{match[1]}#{match[2]}"
-        else; url.url
+    def self.unmobile(url)
+      case
+        when match = TwitterRe.match(url); "#{match[1]}#{match[2]}"
+        when match = WikipediaRe.match(url); "#{match[1]}#{match[2]}"
+        else; url
       end
     end
 
+    def self.run(url, config)
+      url.url = unmobile(url.url)
+      url.via = unmobile(url.via)
+    end
+
   end
 
 end

data/plugins/add_pre_41_unajax_twitter.rb

@@ -0,0 +1,19 @@
+require 'murlsh'
+
+module Murlsh
+
+  # Convert Ajax friendly Twitter urls (with #!) into usable urls.
+  class AddPre41UnajaxTwitter < Plugin
+
+    @hook = 'add_pre'
+
+    TwitterAjaxRe = %r{^(https?://twitter\.com/)#!/}i
+
+    def self.run(url, config)
+      url.url.sub!(TwitterAjaxRe, '\1')
+      url.via.sub!(TwitterAjaxRe, '\1')  if url.via
+    end
+
+  end
+
+end

data/spec/uri_ask_spec.rb

@@ -2,7 +2,7 @@ require 'uri'
 
 require 'murlsh'
 
-Dir['plugins/*.rb'].each { |p| require p }
+Dir['plugins/*.rb'].each { |p| require "./#{p}" }
 
 describe Murlsh::UriAsk do
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: murlsh
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 29
   prerelease: false
   segments: 
   - 1
   - 2
-  - 0
-  version: 1.2.0
+  - 1
+  version: 1.2.1
 platform: ruby
 authors: 
 - Matthew M. Boedicker
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-16 00:00:00 -05:00
+date: 2010-12-23 00:00:00 -05:00
 default_executable: murlsh
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -337,9 +337,24 @@ dependencies:
   type: :development
   version_requirements: *id020
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: rack-test
   prerelease: false
   requirement: &id021 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 0
+        - 5
+        version: "0.5"
+  type: :development
+  version_requirements: *id021
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id022 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -350,7 +365,7 @@ dependencies:
         - 3
         version: "1.3"
   type: :development
-  version_requirements: *id021
+  version_requirements: *id022
 description: url sharing site framework with easy adding, title lookup, atom feed, thumbnails and embedding
 email: matthewm@boedicker.org
 executables: 
@@ -372,6 +387,7 @@ files:
 - lib/murlsh/auth.rb
 - lib/murlsh/build_query.rb
 - lib/murlsh/config_server.rb
+- lib/murlsh/delicious_parse.rb
 - lib/murlsh/dispatch.rb
 - lib/murlsh/doc.rb
 - lib/murlsh/etag_add_encoding.rb
@@ -399,6 +415,7 @@ files:
 - plugins/add_post_50_update_rss.rb
 - plugins/add_post_60_notify_hubs.rb
 - plugins/add_pre_40_convert_mobile.rb
+- plugins/add_pre_41_unajax_twitter.rb
 - plugins/add_pre_50_lookup_content_type_title.rb
 - plugins/add_pre_50_media_thumbnail.rb
 - plugins/add_pre_50_open_graph_image.rb