murlsh 0.2.1

data/README.textile

@@ -0,0 +1,32 @@
+Simple site for a small group of people to share or archive urls.
+
+* looks up url titles
+* adds thumbnails for and jGrowls embedded versions of Flickr, Imageshack, Vimeo and YouTube urls
+* embeds Flash mp3 player for mp3 urls
+* Gravatar support
+* Atom feed
+* regex search
+* looks good on iPhone
+* rack interface
+* plug-in interface
+
+See "http://urls.matthewm.boedicker.org/":http://urls.matthewm.boedicker.org/ for example.
+
+Phusion Passenger Setup:
+
+<pre>
+<code>
+rake gemspec build
+gem install pkg/murlsh-x.x.x.gem
+</code>
+</pre>
+
+In the web directory:
+
+<pre>
+<code>
+murlsh
+edit config.yaml
+rake db:init user:add
+</code>
+</pre>

data/Rakefile

@@ -0,0 +1,192 @@
+$:.unshift(File.join(File.dirname(__FILE__), 'lib'))
+
+require 'rubygems'
+
+require 'murlsh'
+
+require 'flog'
+require 'rake/testtask'
+require 'sqlite3'
+
+require 'pp'
+require 'yaml'
+
+config = YAML.load_file('config.yaml')
+
+desc "Test remote content type fetch for a URL and show errors."
+task :content_type, :url do |t, args|
+  puts Murlsh.get_content_type(args.url, :failproof => false)
+end
+
+namespace :db do
+
+  desc 'Delete the last url added.'
+  task :delete_last_url do
+    ActiveRecord::Base.establish_connection(:adapter => 'sqlite3',
+      :database => config.fetch('db_file'))
+
+    last = Murlsh::Url.find(:last, :order => 'time')
+    pp last
+    response = ask('Delete this url', '?')
+    last.destroy if %w{y yes}.include?(response.downcase)
+  end
+
+  desc "Check for duplicate URLs."
+  task :dupcheck do
+    db = SQLite3::Database.new(config.fetch('db_file'))
+    db.results_as_hash = true
+    h = {}
+    db.execute("SELECT * FROM urls").each do |r|
+      h[r['url']] = h.fetch(r['url'], []).push([r['id'], r['time']])
+    end
+    h.select { |k,v| v.size > 1 }.each do |k,v|
+      puts k
+      v.each { |id,time| puts "  #{id} #{time}" }
+    end
+  end
+
+  desc "Create an empty database."
+  task :init do
+    puts "creating #{config.fetch('db_file')}"
+    db = SQLite3::Database.new(config.fetch('db_file'))
+    db.execute("CREATE TABLE urls (
+      id INTEGER PRIMARY KEY,
+      time TIMESTAMP,
+      url TEXT,
+      email TEXT,
+      name TEXT,
+      title TEXT,
+      content_type TEXT);
+      ")
+  end
+
+  desc 'Interact with the database.'
+  task :shell do
+    exec "sqlite3 #{config['db_file']}"
+  end
+
+end
+
+namespace :dreamhost do
+
+  desc "Restart Passenger."
+  task :restart do
+    open('tmp/restart.txt', 'w') { |f| }
+  end
+
+end
+
+desc "Run flog on ruby and report on complexity."
+task :flog do
+  flog = Flog.new
+  flog.flog('lib')
+  flog.report
+end
+
+desc "Run test suite."
+Rake::TestTask.new do |t|
+  t.pattern = 'test/*_test.rb'
+  t.verbose = true
+  t.warning = true
+end
+
+desc "Test remote title fetch for a URL and show errors."
+task :title, :url do |t, args|
+  puts Murlsh.get_title(args.url, :failproof => false)
+end
+
+desc 'Try to fetch the title for a url and update it in the database.'
+task :title_fetch, :url_id do |t, args|
+  ActiveRecord::Base.establish_connection(:adapter => 'sqlite3',
+    :database => config.fetch('db_file'))
+  url = Murlsh::Url.find(args.url_id)
+  puts "Url: #{url.url}"
+  puts "Previous title: #{url.title}"
+  url.title = Murlsh.get_title(url.url, :failproof => false)
+  url.save
+  puts "\nNew title: #{url.title}"
+end
+
+namespace :user do
+
+  desc "Add a new user."
+  task :add do
+    puts "adding to #{config.fetch('auth_file')}"
+    username = ask(:username)
+    email = ask(:email)
+    password = ask(:password)
+
+    Murlsh::Auth.new(config.fetch('auth_file')).add_user(username, email,
+      password)
+  end
+
+end
+
+desc "Validate XHTML."
+task :validate do
+  require 'cgi'
+  require 'net/http'
+
+  net_http = Net::HTTP.new('validator.w3.org', 80)
+  #net_http.set_debug_output(STDOUT)
+
+  check_url = config.fetch('root_url')
+
+  print "validating #{check_url} : "
+
+  net_http.start do |http|
+    resp = http.request_head(
+      "/check?uri=#{CGI::escape(check_url)}&charset=(detect+automatically)&doctype=Inline&group=0")
+    result = resp['X-W3C-Validator-Status']
+    errors = resp['X-W3C-Validator-Errors']
+    warnings = resp['X-W3C-Validator-Warnings']
+
+    puts "#{result} (#{errors} errors, #{warnings} warnings)"
+  end
+
+end
+
+desc 'Generate a shell script that will post a new url.'
+task :post_sh do
+  puts <<EOS
+#!/bin/sh
+
+URL="$1"
+AUTH="$2" # password can be passed as second parameter or hardcoded here
+
+curl \\
+  --data-urlencode "url=${URL}" \\
+  --data-urlencode "auth=${AUTH}" \\
+  #{config.fetch('root_url')}
+EOS
+end
+
+def ask(prompt, sep=':')
+  print "#{prompt}#{sep} "
+  return STDIN.gets.chomp
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = 'murlsh'
+    gemspec.summary = 'url sharing site framework'
+    gemspec.description = 'url sharing site framework with easy adding, title lookup, atom feed, thumbnails and embedding'
+    gemspec.email = 'matthewm@boedicker.org'
+    gemspec.homepage = 'http://github.com/mmb/murlsh'
+    gemspec.authors = ['Matthew M. Boedicker']
+
+    %w{
+      activerecord 2.3.4
+      bcrypt 2.1.2
+      builder 2.1.2
+      hpricot 0.8.1
+      htmlentities 4.2.0
+      rack 1.0.1
+      sqlite3 1.2.5
+     }.each_slice(2) { |g,v| gemspec.add_dependency(g, ">= #{v}") }
+
+  end
+rescue LoadError
+  puts 'Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com'
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.1

data/bin/murlsh

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+
+FileUtils.cp_r(
+  %w{.htaccess config.ru config.yaml plugins/ public/ Rakefile}.collect { |x| 
+    File.join(File.dirname(__FILE__), '..', x) }, '.', :verbose => true)
+
+FileUtils.mkdir('tmp')
+
+puts <<eos
+Next steps:
+
+edit config.yaml
+rake db:init user:add
+eos

data/config.ru

@@ -0,0 +1,11 @@
+$:.unshift(File.join(File.dirname(__FILE__), 'lib'))
+
+require 'murlsh'
+
+# use Rack::ShowExceptions
+use Rack::ConditionalGet
+use Rack::Deflater
+use Rack::Static, :urls => %w{/css /js /swf}, :root => 'public'
+use Rack::Static, :urls => %w{/atom.xml}
+
+run Murlsh::Dispatch.new

data/config.yaml

@@ -0,0 +1,18 @@
+--- 
+auth_file: /home/mmb/src/murlsh/murlsh_users
+description: URLs found interesting by Matthew M. Boedicker
+db_file: murlsh.db
+feed_file: atom.xml
+google_verify: 
+gravatar_size: 32
+num_posts_feed: 25
+num_posts_page: 100
+page_title: mmb url share
+root_url: http://urls.matthewm.boedicker.org/
+css_prefix: css/
+img_prefix: img/
+js_prefix: js/
+swf_prefix: swf/
+css_files:
+- jquery.jgrowl.css
+- screen.css

data/lib/murlsh/atom_feed.rb

@@ -0,0 +1,76 @@
+require 'rubygems'
+require 'builder'
+
+require 'uri'
+
+module Murlsh
+
+  class AtomFeed
+
+    def initialize(root_url, options={})
+      options = {
+        :filename => 'atom.xml',
+        :title => 'Atom feed' }.merge(options)
+      @root_url = root_url
+      @filename = options[:filename]
+      @title = options[:title]
+
+      setup_id_fields
+    end
+
+    def setup_id_fields
+      uri_parsed = URI(@root_url)
+
+      m = uri_parsed.host.match(/^(.*?)\.?([^.]+\.[^.]+)$/)
+
+      @host, @domain = (m ? m.captures : [uri_parsed.host, ''])
+
+      @path = uri_parsed.path
+    end
+
+    def write(entries, path)
+      open(path, 'w') do |f|
+        f.flock(File::LOCK_EX)
+
+        make(entries, :target => f)
+
+        f.flock(File::LOCK_UN)
+      end
+    end
+
+    def make(entries, options={})
+      xm = Builder::XmlMarkup.new(options)
+      xm.instruct! :xml
+
+      xm.feed(:xmlns => 'http://www.w3.org/2005/Atom') {
+        xm.id(@root_url)
+        xm.link(:href => URI.join(@root_url, @filename), :rel => 'self')
+        xm.title(@title)
+        xm.updated(entries.collect { |mu| mu.time }.max.xmlschema)
+        entries.each do |mu|
+          xm.entry {
+            xm.author { xm.name(mu.name) }
+            xm.title(mu.title)
+            xm.id(entry_id(mu))
+            xm.summary(mu.title)
+            xm.updated(mu.time.xmlschema)
+            xm.link(:href => mu.url)
+            enclosure(xm, mu)
+          }
+        end
+      }
+      xm
+    end
+
+    def entry_id(url)
+      "tag:#{@domain},#{url.time.strftime('%Y-%m-%d')}:#{@host}#{@path}#{url.id}"
+    end
+
+    def enclosure(xm, mu)
+      xm.link(:rel => 'enclosure', :type => mu.content_type, :href => mu.url,
+        :title => 'Full-size') if mu.is_image?
+    end
+
+  end
+
+end

data/lib/murlsh/auth.rb

@@ -0,0 +1,33 @@
+require 'rubygems'
+require 'bcrypt'
+
+require 'csv'
+require 'digest/md5'
+
+module Murlsh
+
+  class Auth
+
+    def initialize(file)
+      @file = file
+    end
+
+    def auth(password)
+      CSV::Reader.parse(open(@file)) do |row|
+        return { :name => row[0], :email => row[1] } if
+          BCrypt::Password.new(row[2]) == password
+      end
+    end
+
+    def add_user(username, email, password)
+      open(@file, 'a') do |f|
+        f.flock(File::LOCK_EX)
+        f.write("#{[username, Digest::MD5.hexdigest(email),
+          BCrypt::Password.create(password)].join(',')}\n")
+        f.flock(File::LOCK_UN)
+      end
+    end
+
+  end
+
+end

data/lib/murlsh/dispatch.rb

@@ -0,0 +1,54 @@
+%w{
+murlsh
+
+rubygems
+active_record
+rack
+sqlite3
+
+yaml
+}.each { |m| require m }
+
+module Murlsh
+
+  class Dispatch
+
+    def initialize
+      @config = YAML.load_file('config.yaml')
+      @url_root = URI(@config.fetch('root_url')).path
+
+      ActiveRecord::Base.establish_connection(
+        :adapter => 'sqlite3', :database => @config.fetch('db_file'))
+
+      @db = ActiveRecord::Base.connection.instance_variable_get(:@connection)
+
+      @url_server = Murlsh::UrlServer.new(@config, @db)
+    end
+
+    def call(env)
+      dispatch = {
+        ['GET', @url_root] => [@url_server, :get],
+        ['POST', @url_root] => [@url_server, :post],
+        ['GET', "#{@url_root}url"] => [@url_server, :get],
+        ['POST', "#{@url_root}url"] => [@url_server, :post],
+      }
+      dispatch.default = [self, :not_found]
+
+      req = Rack::Request.new(env)
+
+      obj, meth = dispatch[[req.request_method, req.path]]
+
+      obj.send(meth, req).finish
+    end
+
+    def not_found(req)
+      Rack::Response.new("<p>#{req.url} not found</p>
+
+<p><a href=\"#{@config['root_url']}\">root<a></p>
+",
+        404, { 'Content-Type' => 'text/html' })
+    end
+
+  end
+
+end

data/lib/murlsh/get_content_type.rb

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'net/https'
+require 'uri'
+
+class URI::Generic
+
+  def path_query
+    path + (query ? "?#{query}" : '')
+  end
+
+end
+
+module Murlsh
+
+  module_function
+
+  def get_content_type(url, options={})
+    options[:headers] = default_headers(url).merge(
+      options.fetch(:headers, {}))
+
+    options = {
+      :failproof => true,
+      :redirects => 0,
+      }.merge(options)
+
+    unless options[:redirects] > 3
+      begin
+        url = parse_uri(url)
+
+        make_net_http(url, options).start do |http|
+          resp = get_resp(http, url, options[:headers])
+          case resp
+            when Net::HTTPSuccess then return resp['content-type']
+            when Net::HTTPRedirection then
+              options[:redirects] += 1
+              return get_content_type(resp['location'], options)
+          end
+        end
+      rescue Exception => e
+        raise unless options[:failproof]
+      end
+    end
+    ''
+  end
+
+  # Parse a URI if it's not already parsed.
+  def parse_uri(uri)
+    uri.is_a?(URI::HTTP) ? uri : URI(uri)
+  end
+
+  def make_net_http(url, options={})
+    net_http = Net::HTTP.new(url.host, url.port)
+    net_http.use_ssl = (url.scheme == 'https')
+    net_http.set_debug_output(options[:debug]) if options[:debug]
+    net_http
+  end
+
+  # Get the response to HTTP HEAD. If HEAD not allowed do GET.
+  def get_resp(http, url, headers={})
+    resp = http.request_head(url.path_query, headers)
+    if Net::HTTPMethodNotAllowed === resp
+      http.request_get(url.path_query, headers)
+    else
+      resp
+    end
+  end
+
+  def default_headers(url)
+    result = {
+      'User-Agent' =>
+        'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
+    }
+    begin
+      parsed_url = parse_uri(url)
+      if (parsed_url.host || '')[/^www\.nytimes\.com/]
+        result['Referer'] = 'http://news.google.com/'
+      end
+    rescue URI::InvalidURIError => e
+    end
+
+    result
+  end
+
+end

data/lib/murlsh/get_title.rb

@@ -0,0 +1,65 @@
+require 'rubygems'
+require 'hpricot'
+require 'htmlentities'
+
+require 'iconv'
+require 'open-uri'
+require 'uri'
+
+module Murlsh
+
+  module_function
+
+  def get_title(url, options={})
+    options[:headers] = default_headers(url).merge(
+      options.fetch(:headers, {}))
+
+    options = {
+      :failproof => true,
+      }.merge(options)
+
+    result = nil
+    begin
+      options[:content_type] ||= get_content_type(url, options)
+      if might_have_title(options[:content_type])
+        f = open(url, options[:headers])
+
+        doc = Hpricot(f)
+
+        result = HTMLEntities.new.decode(Iconv.conv('utf-8',
+          get_charset(doc) || f.charset, find_title(doc)))
+      end
+    rescue Exception => e
+       raise unless options[:failproof]
+    end
+    (result and !result.empty?) ? result : url
+  end
+
+  def might_have_title(content_type)
+    content_type[/^text\/html/]
+  end
+
+  # Find the title in an Hpricot document.
+  def find_title(doc)
+    %w{//html/head/title //head/title //html/title //title}.each do |xpath|
+      return (doc/xpath).first.inner_html unless (doc/xpath).first.nil?
+    end
+    nil
+  end
+
+  # Get the character set of an Hpricot document.
+  def get_charset(doc)
+    %w{content-type Content-Type}.each do |ct|
+      content_type = doc.at("meta[@http-equiv='#{ct}']")
+      unless content_type.nil?
+        content = content_type['content']
+        unless content.nil?
+          charset = content[/charset=([\w_.:-]+)/, 1]
+          return charset if charset
+        end
+      end
+    end
+    nil
+  end
+
+end

data/lib/murlsh/markup.rb

@@ -0,0 +1,97 @@
+module Murlsh
+
+  module Markup
+
+    def javascript(sources, options={})
+      sources.to_a.each do |src|
+        script('', :type => 'text/javascript',
+          :src => "#{options[:prefix]}#{src}")
+      end
+    end
+
+    def murlsh_img(options={})
+      img_convert_prefix(options)
+      img_convert_size(options)
+      img_convert_text(options)
+
+      if options[:href]
+        a(:href => options[:href]) {
+          options.delete(:href)
+          img(options)
+        }
+      else
+        img(options)
+      end
+    end
+
+    def atom(href)
+      link(:rel => 'alternate', :type => 'application/atom+xml', :href => href)
+    end
+
+    def css(hrefs, options={})
+      hrefs.to_a.each do |href|
+        attrs = {
+          :href => "#{options[:prefix]}#{href}",
+          :rel => 'stylesheet',
+          :type => 'text/css',
+        }
+        attrs[:media] = options[:media] if options[:media]
+        link(attrs)
+      end
+    end
+
+    def metas(tags)
+      tags.each { |k,v| meta(:name => k, :content => v) }
+    end
+
+    def gravatar(email_hash, options={})
+      query = options.reject do |k,v|
+        not ((k == 'd' and %w{identicon monsterid wavatar}.include?(v)) or
+        (k =='s' and (0..512).include?(v)) or
+        (k == 'r' and %w{g pg r x}.include?(v)))
+      end
+
+      return if query['s'] and query['s'] < 1
+
+      options.reject! { |k,v| %w{d s r}.include?(k) }
+      options[:src] = URI.join('http://www.gravatar.com/avatar/',
+        email_hash + build_query(query))
+
+      murlsh_img(options)
+    end
+
+    def build_query(h)
+      h.empty? ? '' :
+        '?' + h.collect { |k,v| URI.escape("#{k}=#{v}") }.join('&')
+    end
+
+    private
+
+    def img_convert_prefix(options)
+      if options.has_key?(:prefix) and options.has_key?(:src)
+        options[:src] = options[:prefix] + options[:src]
+        options.delete(:prefix)
+      end
+    end
+
+    def img_convert_size(options)
+      if options.has_key?(:size)
+        if options[:size].kind_of?(Array) and options[:size].size == 2
+          options[:width], options[:height] = options[:size]
+        else
+          options[:width] = options[:height] = options[:size]
+        end
+        options.delete(:size)
+      end
+    end
+
+    def img_convert_text(options)
+      if options.has_key?(:text)
+        options[:alt] = options[:title] = options[:text]
+        options.delete(:text)
+      end
+    end
+
+  end
+
+end