murlsh 1.0.0 → 1.1.0

data/.htaccess

@@ -12,3 +12,8 @@ AddOutputFilterByType DEFLATE text/html
   Header add Expires "Wed, 22 Jun 2019 20:07:00 GMT"
   FileETag None
 </FilesMatch>
+
+<FilesMatch "[\da-z]{32}\.(gif|jpe?g|png)$">
+  Header add Expires "Wed, 22 Jun 2019 20:07:00 GMT"
+  FileETag None
+</FilesMatch>

data/COPYING

@@ -1,3 +1,30 @@
+Copyright (c) 2010 Matthew M. Boedicker
+
+All code is licensed under the GPLv3 with the following exceptions, which are
+included third-party libraries:
+
+- jQuery by John Resig
+
+License: MIT or GPLv2
+
+Files:
+public/js/jquery-1.4.4.min.js
+
+- jGrowl by Stan Lemon
+
+License: MIT or GPLv2
+
+Files:
+public/css/jquery.jgrowl.css
+public/js/jquery.jgrowl_compressed.js
+
+- twitter-text-js by Twitter, Inc.
+
+License: Apache 2.0
+
+Files:
+public/js/twitter-text-1.0.4.js
+
                     GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 

data/README.textile

@@ -1,7 +1,7 @@
 Site for sharing and archiving links.
 
-* looks up url titles
-* adds thumbnails for and jGrowls embedded versions of Imageshack, Vimeo and YouTube urls
+* fetches url titles and generates thumbnails
+* jGrowls embedded versions of Imageshack, Vimeo and YouTube urls
 * converts Twitter status urls to their full text and adds user thumbnail
 * generates Atom and RSS feeds
 * regex search

data/Rakefile

@@ -1,24 +1,20 @@
 $:.unshift(File.join(File.dirname(__FILE__), 'lib'))
 
-%w{
-cgi
-digest/md5
-net/http
-pp
-uri
-yaml
-
-flog
-spec/rake/spectask
-sqlite3
-
-murlsh
-}.each { |d| require d }
-
-# optional libraries
-%w{
-metric_fu
-}.each { |d| Murlsh::failproof { require d } }
+require 'cgi'
+require 'digest/md5'
+require 'net/http'
+require 'pp'
+require 'uri'
+require 'yaml'
+
+require 'RMagick'
+require 'sqlite3'
+
+require 'murlsh'
+
+def gem_not_found(gem_name)
+  puts "#{gem_name} not found, install it with: gem install #{gem_name}"
+end
 
 config = YAML.load_file('config.yaml')
 
@@ -37,7 +33,7 @@ end
 desc 'Combine and compress static files.'
 task :compress => %w{css:compress js:compress}
 
-desc "Test remote content type fetch for a URL and show errors."
+desc 'Test remote content type fetch for a URL and show errors.'
 task :content_type, :url do |t, args|
   puts URI(args.url).extend(Murlsh::UriAsk).content_type(:failproof => false,
     :debug => STDOUT)
@@ -53,28 +49,28 @@ namespace :db do
     last = Murlsh::Url.find(:last, :order => 'time')
     pp last
     response = ask('Delete this url', '?')
-    last.destroy if %w{y yes}.include?(response.downcase)
+    last.destroy  if %w{y yes}.include?(response.downcase)
   end
 
-  desc "Check for duplicate URLs."
+  desc 'Check for duplicate URLs.'
   task :dupcheck do
     db = SQLite3::Database.new(config.fetch('db_file'))
     db.results_as_hash = true
     h = {}
-    db.execute("SELECT * FROM urls").each do |r|
+    db.execute('SELECT * FROM urls').each do |r|
       h[r['url']] = h.fetch(r['url'], []).push([r['id'], r['time']])
     end
-    h.select { |k,v| v.size > 1 }.each do |k,v|
+    h.find_all { |k,v| v.size > 1 }.each do |k,v|
       puts k
       v.each { |id,time| puts "  #{id} #{time}" }
     end
   end
 
-  desc "Create an empty database."
+  desc 'Create an empty database.'
   task :init do
     puts "creating #{config.fetch('db_file')}"
     db = SQLite3::Database.new(config.fetch('db_file'))
-    db.execute("CREATE TABLE urls (
+    db.execute('CREATE TABLE urls (
       id INTEGER PRIMARY KEY,
       time TIMESTAMP,
       url TEXT,
@@ -85,7 +81,7 @@ namespace :db do
       content_type TEXT,
       via TEXT,
       thumbnail_url TEXT);
-      ")
+      ')
   end
 
   desc 'Interact with the database.'
@@ -93,6 +89,17 @@ namespace :db do
     exec "sqlite3 #{config['db_file']}"
   end
 
+  desc 'Search urls and titles in the database.'
+  task :grep, :search do |t,args|
+    ActiveRecord::Base.establish_connection(:adapter => 'sqlite3',
+      :database => config.fetch('db_file'))
+
+    Murlsh::Url.all(:conditions => [
+      'MURLSHMATCH(title, :search) OR MURLSHMATCH(url, :search)',
+      { :search => args.search }]).each do |url|
+      puts "#{url.id} #{url.url} #{url.title}"
+    end
+  end
 end
 
 directory 'tmp'
@@ -106,29 +113,43 @@ namespace :passenger do
 
 end
 
-desc "Run flog on ruby and report on complexity."
+desc 'Run flog on ruby and report on complexity.'
 task :flog do
-  flog = Flog.new
-  flog.flog('lib')
-  flog.report
+  begin
+    require 'flog'
+
+    flog = Flog.new
+    flog.flog 'lib'
+    flog.report
+  rescue LoadError
+    gem_not_found 'flog'
+  end
 end
 
-desc "Run test suite."
-Spec::Rake::SpecTask.new('test') do |t|
-  t.spec_files = FileList['spec/*_spec.rb']
-  t.spec_opts = %w{--color}
-  # list of places to check for unicode_formatter.rb and use it if found
-  %w{unicode_formatter.rb}.map { |x| File.expand_path(x) }.each do |f|
-    if File.exists?(f)
-      t.spec_opts.push(*%W{--require #{f} --format UnicodeFormatter})
-      break
+desc 'Run test suite.'
+begin
+  require 'spec/rake/spectask'
+
+  Spec::Rake::SpecTask.new('test') do |t|
+    t.spec_files = FileList['spec/*_spec.rb']
+    t.spec_opts = %w{--color}
+    # list of places to check for unicode_formatter.rb and use it if found
+    %w{unicode_formatter.rb}.map { |x| File.expand_path(x) }.each do |f|
+      if File.exists?(f)
+        t.spec_opts.push(*%W{--require #{f} --format UnicodeFormatter})
+        break
+      end
     end
+    t.verbose = true
+    t.warning = true
+  end
+rescue LoadError
+  task :test do
+    gem_not_found 'rspec'
   end
-  t.verbose = true
-  t.warning = true
 end
 
-desc "Test remote title fetch for a URL and show errors."
+desc 'Test remote title fetch for a URL and show errors.'
 task :title, :url do |t, args|
   puts URI(args.url).extend(Murlsh::UriAsk).title(:failproof => false,
     :debug => STDOUT)
@@ -148,7 +169,7 @@ end
 
 namespace :user do
 
-  desc "Add a new user."
+  desc 'Add a new user.'
   task :add do
     puts "adding to #{config.fetch('auth_file')}"
     username = ask(:username)
@@ -168,7 +189,7 @@ def validate(check_url, options={})
     :validator_port => 80,
     :validator_path =>
       "/check?uri=#{CGI::escape(check_url)}&charset=(detect+automatically)&doctype=Inline&group=0",
-  }.merge(options)
+  }.merge options
 
   net_http = Net::HTTP.new(opts[:validator_host], opts[:validator_port])
   # net_http.set_debug_output(STDOUT)
@@ -229,7 +250,7 @@ def cat(in_files, sep=nil)
   in_files.each do |fname|
     open(fname) do |h|
       while (line = h.gets) do; result << line; end
-      result << sep if sep
+      result << sep  if sep
     end
   end
   result
@@ -319,9 +340,49 @@ namespace :js do
 
 end
 
+namespace :thumb do
+
+  desc 'Check that local thumbnails in database are consistent with filesystem.'
+  task :check do
+    ActiveRecord::Base.establish_connection :adapter => 'sqlite3',
+      :database => config.fetch('db_file')
+    Murlsh::Url.all(
+      :conditions => "thumbnail_url like 'img/thumb/%'").each do |u|
+      identity = "url #{u.id} (#{u.url})"
+
+      path = File.join(%w{public}.concat(File.split(u.thumbnail_url)))
+      if File.readable?(path)
+        img_data = open(path) { |f| f.read }
+
+        unless img_data.empty?
+          img = Magick::ImageList.new.from_blob(img_data).extend(
+            Murlsh::ImageList)
+
+          ext = File.extname(path)
+          expected_ext = img.preferred_extension
+          if ext != expected_ext
+            puts "#{identity} thumbnail #{path} has an extension of '#{ext}' but is actually a '#{expected_ext}'"
+
+          end
+
+          md5 = Digest::MD5.hexdigest(img_data)
+          if File.basename(path, ext) != md5
+            puts "#{identity} thumbnail #{path} filename does not match file content md5 (#{md5})"
+          end
+        else
+          puts "#{identity} thumbnail #{path} is empty"
+        end
+      else
+        puts "#{identity} thumbnail #{path} does not exist or is not readable"
+      end
+    end
+  end
+
+end
+
 def ask(prompt, sep=':')
   print "#{prompt}#{sep} "
-  return STDIN.gets.chomp
+  STDIN.gets.chomp
 end
 
 begin
@@ -339,25 +400,31 @@ begin
     # gemspec.cert_chain = %w{/home/mmb/src/keys/gem-public_cert.pem}
 
     %w{
-      activerecord 2.3.4
-      bcrypt-ruby 2.1.2
-      builder 2.1.2
-      flickraw 0.8.3
-      flog 2.5.0
-      hpricot 0.8.1
-      htmlentities 4.2.0
-      json 1.2.3
-      push-notify 0.1.0
-      rack 1.0.0
-      rack-cache 0.5.2
-      rack-rewrite 1.0.2
-      rack-throttle 0.3.0
-      sqlite3-ruby 1.2.1
-      tinyatom 0.2.0
-      twitter 0.9.12
-      vimeo 1.2.2
-      }.each_slice(2) { |g,v| gemspec.add_dependency(g, ">= #{v}") }
-      gemspec.add_dependency('rspec',  '~> 1.3')
+      activerecord >= 2.3.4
+      bcrypt-ruby >= 2.1.2
+      builder >= 2.1.2
+      flickraw >= 0.8.3
+      hpricot >= 0.8.1
+      htmlentities >= 4.2.0
+      json >= 1.2.3
+      plumnailer >= 0.1.0
+      push-notify >= 0.1.0
+      rack >= 1.0.0
+      rack-cache >= 0.5.2
+      rack-rewrite >= 1.0.2
+      rack-throttle >= 0.3.0
+      rmagick >= 1.15.14
+      sqlite3-ruby >= 1.2.1
+      tinyatom >= 0.3.3
+      twitter >= 0.9.12
+      vimeo >= 1.2.2
+      }.each_slice(3) { |g,o,v| gemspec.add_dependency(g, "#{o} #{v}") }
+    %w{
+      flog >= 2.5.0
+      rspec ~> 1.3
+      }.each_slice(3) do |g,o,v|
+      gemspec.add_development_dependency(g, "#{o} #{v}")
+    end
   end
 rescue LoadError
   puts "Jeweler not available. Install it with: gem install jeweler"

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0

data/config.ru

@@ -1,14 +1,12 @@
 $:.unshift(File.join(File.dirname(__FILE__), 'lib'))
 
-%w{
-yaml
+require 'yaml'
 
-rack/cache
-rack/rewrite
-rack/throttle
+require 'rack/cache'
+require 'rack/rewrite'
+require 'rack/throttle'
 
-murlsh
-}.each { |m| require m }
+require 'murlsh'
 
 config = YAML.load_file('config.yaml')
 
@@ -24,7 +22,10 @@ end
 use Rack::ConditionalGet
 use Murlsh::EtagAddEncoding
 use Rack::Deflater
-use Murlsh::FarFutureExpires, :patterns => %r{\.gen\.(css|js)$}
+use Murlsh::FarFutureExpires, :patterns => [
+  %r{[\da-z]{32}\.(?:gif|jpe?g|png)$}i,
+  %r{\.gen\.(css|js)$}
+]
 
 feed_url = URI.join(config.fetch('root_url'), config.fetch('feed_file'))
 use Murlsh::MustRevalidate, :patterns => %r{^#{Regexp.escape(feed_url.path)}$}

data/config.yaml

@@ -12,9 +12,9 @@ feed_file: atom.atom
 flickr_api_key: 
 gravatar_size: 32
 js_files: 
-- js/jquery-1.4.3.min.js
+- js/jquery-1.4.4.min.js
 - js/jquery.jgrowl_compressed.js
-- js/twitter-text-1.0.3.js
+- js/twitter-text-1.0.4.js
 - js/js.js
 meta_tag_description: URLs found interesting by Matthew M. Boedicker
 meta_tag_verify-v1: 
@@ -26,4 +26,5 @@ pubsubhubbub_hubs: []
 
 root_url: http://urls.matthewm.boedicker.org/
 show_names: true
+thumbnail_max_side: 90
 user_agent: murlsh (http://github.com/mmb/murlsh)

data/lib/murlsh/auth.rb

@@ -1,9 +1,7 @@
-%w{
-csv
-digest/md5
+require 'csv'
+require 'digest/md5'
 
-bcrypt
-}.each { |m| require m }
+require 'bcrypt'
 
 module Murlsh
 
@@ -22,7 +20,7 @@ module Murlsh
     # Authenticate a user by password. Return their name and email if correct.
     def auth(password)
       CSV::Reader.parse(open(@file)) do |row|
-        return { :name => row[0], :email => row[1] } if
+        return { :name => row[0], :email => row[1] }  if
           BCrypt::Password.new(row[2]) == password
       end
     end
@@ -30,8 +28,8 @@ module Murlsh
     # Add a user to the authentication file.
     def add_user(username, email, password)
       Murlsh::openlock(@file, 'a') do |f|
-        f.write("#{[username, Digest::MD5.hexdigest(email),
-          BCrypt::Password.create(password)].join(',')}\n")
+        f.write "#{[username, Digest::MD5.hexdigest(email),
+          BCrypt::Password.create(password)].join(',')}\n"
       end
     end
 

data/lib/murlsh/config_server.rb

@@ -1,9 +1,7 @@
-%w{
-digest/sha1
+require 'digest/sha1'
 
-json
-rack
-}.each { |m| require m }
+require 'json'
+require 'rack'
 
 module Murlsh
 
@@ -27,7 +25,7 @@ module Murlsh
     end
 
     # Serve a JSON subset of the configuration.
-    def get(req); Rack::Response.new(@config_json, 200, @headers); end
+    def get(req); Rack::Response.new @config_json, 200, @headers; end
 
   end
 

data/lib/murlsh/dispatch.rb

@@ -1,9 +1,7 @@
-%w{
-active_record
-rack
+require 'active_record'
+require 'rack'
 
-murlsh
-}.each { |m| require m }
+require 'murlsh'
 
 module Murlsh
 
@@ -51,11 +49,11 @@ module Murlsh
 
     # Called if the request is not found.
     def not_found(req)
-      Rack::Response.new("<p>#{req.url} not found</p>
+      Rack::Response.new "<p>#{req.url} not found</p>
 
 <p><a href=\"#{@config['root_url']}\">root<a></p>
 ",
-        404, { 'Content-Type' => 'text/html' })
+        404, { 'Content-Type' => 'text/html' }
     end
 
   end

data/lib/murlsh/doc.rb

@@ -11,7 +11,7 @@ module Murlsh
           content = content_type['content']
           unless content.nil?
             charset = content[/charset=([\w_.:-]+)/, 1]
-            return charset if charset
+            return charset  if charset
           end
         end
       end

data/lib/murlsh/etag_add_encoding.rb

@@ -1,6 +1,4 @@
-%w{
-rack/utils
-}.each { |m| require m }
+require 'rack/utils'
 
 module Murlsh
 

data/lib/murlsh/failproof.rb

@@ -1,4 +1,3 @@
-
 module Murlsh
 
   module_function

data/lib/murlsh/far_future_expires.rb

@@ -1,7 +1,5 @@
-%w{
-rack
-rack/utils
-}.each { |m| require m }
+require 'rack'
+require 'rack/utils'
 
 module Murlsh
 

data/lib/murlsh/head_from_get.rb

@@ -1,9 +1,9 @@
 module Murlsh
 
-  # mixin for adding head() that calls get() and removed the body
+  # Mixin for adding head() that calls get() and removed the body.
   module HeadFromGet
 
-    # call get() and remove the body
+    # Call get() and remove the body.
     def head(req)
       resp = get(req)
       resp.body = ''

data/lib/murlsh/image_list.rb

@@ -0,0 +1,32 @@
+require 'base64'
+
+module Murlsh
+
+  # Magick::ImageList mixin.
+  module ImageList
+
+    # For each image, if the width or height is larger than max_side, resize so
+    # that the longest side = max_side.
+    def resize_down!(max_side)
+      each do |i|
+        if i.columns > max_side or i.rows > max_side
+          i.resize_to_fit! max_side, max_side
+        end
+        i.strip!
+      end
+    end
+
+    # Get the preferred extension for this image.
+    def preferred_extension; FormatExtensions[self.format]; end
+
+    def data_uri; "data:#{mime_type};base64,#{Base64.encode64(to_blob)}"; end
+
+    FormatExtensions = {
+      'GIF' => '.gif',
+      'JPEG' => '.jpg',
+      'PNG' => '.png',
+    }
+
+  end
+
+end

data/lib/murlsh/img_store.rb

@@ -1,13 +1,20 @@
-%w{
-cgi
-open-uri
-}.each { |m| require m }
+require 'cgi'
+require 'digest/md5'
+require 'open-uri'
+require 'uri'
+
+require 'RMagick'
+
+require 'murlsh'
 
 module Murlsh
 
   # Fetch images from urls and store them locally.
   class ImgStore
 
+    # Fetch images from urls and store them locally.
+    # Options:
+    # * :user_agent - user agent to send with http requests
     def initialize(storage_dir, options={})
       @storage_dir = storage_dir
       @user_agent = options[:user_agent]
@@ -16,16 +23,47 @@ module Murlsh
     # Build headers to send with request.
     def headers
       result = {}
-      result['User-Agent'] = @user_agent if @user_agent
+      result['User-Agent'] = @user_agent  if @user_agent
       result
     end
 
     # Fetch an image from a url and store it locally.
-    def store(url)
-      local_file = CGI.escape(url)
+    #
+    # The filename will be the md5sum of the contents plus the correct
+    # extension.
+    #
+    # If a block is given the Magick::ImageList created will be yielded
+    # before storage.
+    def store_url(url, &block)
+      open(url, headers) { |fin| store_img_data fin.read, &block }
+    end
+
+    # Accept a blob of image data and store it locally.
+    #
+    # The filename will be the md5sum of the contents plus the correct
+    # extension.
+    #
+    # If a block is given the Magick::ImageList created will be yielded
+    # before storage.
+    def store_img_data(img_data, &block)
+      img = Magick::ImageList.new.from_blob(img_data)
+      yield img if block_given?
+      store_img img
+    end
+
+    # Accept a Magick::ImageList and store it locally.
+    #
+    # The filename will be the md5sum of the contents plus the correct
+    # extension.
+    def store_img(img)
+      img.extend(Murlsh::ImageList)  unless img.is_a?(Murlsh::ImageList)
+      img_data = img.to_blob
+      md5 = Digest::MD5.hexdigest(img_data)
+
+      local_file = "#{md5}#{img.preferred_extension}"
       local_path = File.join(storage_dir, local_file)
-      open(url, headers) do |fin|
-        open(local_path, 'w') { |fout| fout.write(fin.read) }
+      unless File.exists?(local_path)
+        Murlsh::openlock(local_path, 'w') { |fout| fout.write img_data }
       end
       local_file
     end