murlsh 1.4.1 → 1.5.0

data/lib/murlsh/atom_server.rb

@@ -0,0 +1,38 @@
+require 'uri'
+
+require 'rack'
+
+require 'murlsh'
+
+module Murlsh
+
+  # Serve Atom feed.
+  class AtomServer
+
+    def initialize(config); @config = config; end
+
+    # Respond to a GET request for Atom feed.
+    def get(req)
+      conditions = Murlsh::SearchConditions.new(req['q']).conditions
+      page = 1
+      per_page = config.fetch('num_posts_feed', 25)
+
+      result_set = Murlsh::UrlResultSet.new(conditions, page, per_page)
+      urls = result_set.results
+
+      feed_url = URI.join(config.fetch('root_url'), config.fetch('feed_file'))
+      body = Murlsh::AtomBody.new(config, req, feed_url, urls)
+
+      resp = Rack::Response.new(body, 200,
+        'Cache-Control' => 'must-revalidate, max-age=0',
+        'Content-Type' => 'application/atom+xml')
+      if u = body.updated
+        resp['Last-Modified'] = u.httpdate
+      end
+      resp
+    end
+
+    attr_reader :config
+  end
+
+end

data/lib/murlsh/auth.rb

@@ -44,6 +44,18 @@ module Murlsh
       end
     end
 
+    # Look up a user by email address.
+    #
+    # Return nil if not found.
+    def by_email(email)
+      hash = Digest::MD5.hexdigest(email)
+
+      self.class.csv_iter(@file) do |row|
+        return { :name => row[0], :email => row[1] }  if hash == row[1]
+      end
+
+    end
+
   end
 
 end

data/lib/murlsh/build_query.rb

@@ -4,7 +4,7 @@ module Murlsh
 
   # Query string builder. Takes hash of query string variables.
   def build_query(h)
-    h.empty? ? '' : '?' + h.map { |k,v| URI.escape "#{k}=#{v}" }.join('&')
+    h.empty? ? '' : '?' + h.map { |k,v| URI.escape("#{k}=#{v}") }.join('&')
   end
 
 end

data/lib/murlsh/cat_files.rb

@@ -0,0 +1,17 @@
+module Murlsh
+
+  module_function
+
+  # Concatenate some files and return the result as a string.
+  def cat_files(files, sep=nil)
+    result = ''
+    files.each do |fname|
+      open(fname) do |h|
+        while (line = h.gets) do; result << line; end
+        result << sep  if sep
+      end
+    end
+    result
+  end
+
+end

data/lib/murlsh/delicious_parse.rb

@@ -1,4 +1,5 @@
 require 'open-uri'
+require 'time'
 require 'uri'
 
 require 'nokogiri'

data/lib/murlsh/dispatch.rb

@@ -14,25 +14,34 @@ module Murlsh
     def initialize(config)
       @config = config
 
-      url_server = Murlsh::UrlServer.new(config)
+      atom_server = Murlsh::AtomServer.new(config)
       json_server = Murlsh::JsonServer.new(config)
+      m3u_server = Murlsh::M3uServer.new(config)
+      podcast_server = Murlsh::PodcastServer.new(config)
+      pop_server = Murlsh::PopServer.new(config)
+      random_server = Murlsh::RandomServer.new(config)
+      rss_server = Murlsh::RssServer.new(config)
+      url_server = Murlsh::UrlServer.new(config)
+
       root_path = URI(config.fetch('root_url')).path
 
       @routes = [
-        [%r{^HEAD #{root_path}(url)?$}, url_server.method(:head)],
-        [%r{^GET #{root_path}(url)?$}, url_server.method(:get)],
+        [%r{^(?:HEAD|GET) #{root_path}atom\.atom$}, atom_server.method(:get)],
+        [%r{^(?:HEAD|GET) #{root_path}json\.json$}, json_server.method(:get)],
+        [%r{^(?:HEAD|GET) #{root_path}m3u\.m3u$}, m3u_server.method(:get)],
+        [%r{^(?:HEAD|GET) #{root_path}podcast\.rss$}, podcast_server.method(:get)],
+        [%r{^POST #{root_path}pop$}, pop_server.method(:post)],
+        [%r{^(?:HEAD|GET) #{root_path}random$}, random_server.method(:get)],
+        [%r{^(?:HEAD|GET) #{root_path}rss\.rss$}, rss_server.method(:get)],
+        [%r{^(?:HEAD|GET) #{root_path}(url)?$}, url_server.method(:get)],
         [%r{^POST #{root_path}(url)?$}, url_server.method(:post)],
-        [%r{^HEAD #{root_path}json\.json$}, json_server.method(:head)],
-        [%r{^GET #{root_path}json\.json$}, json_server.method(:get)],
       ]
 
       db_init
     end
 
     def db_init
-      ActiveRecord::Base.establish_connection(
-        :adapter => 'sqlite3', :database => @config.fetch('db_file'))
-
+      ActiveRecord::Base.establish_connection config.fetch('db')
       ActiveRecord::Base.default_timezone = :utc
       ActiveRecord::Base.include_root_in_json = false
       # ActiveRecord::Base.logger = Logger.new(STDERR)
@@ -55,13 +64,18 @@ module Murlsh
 
     # Called if the request is not found.
     def not_found(req)
-      Rack::Response.new "<p>#{req.url} not found</p>
+      if req.head?
+        Rack::Response.new([], 404)
+      else
+        Rack::Response.new("<p>#{req.url} not found</p>
 
-<p><a href=\"#{@config['root_url']}\">root<a></p>
+<p><a href=\"#{config.fetch('root_url')}\">root<a></p>
 ",
-        404, { 'Content-Type' => 'text/html' }
+          404, { 'Content-Type' => 'text/html' })
+      end
     end
 
+    attr_reader :config
     attr_accessor :routes
   end
 

data/lib/murlsh/etag_add_encoding.rb

@@ -14,7 +14,7 @@ module Murlsh
       headers = Rack::Utils::HeaderHash.new(headers)
 
       if headers['ETag']
-        headers['ETag'].sub!(/(")?$/, "#{headers['Content-Encoding']}\\1")
+        headers['ETag'].sub! /(")?$/, "#{headers['Content-Encoding']}\\1"
       end
 
       [status, headers, body]

data/lib/murlsh/feed_body.rb

@@ -0,0 +1,36 @@
+module Murlsh
+
+  # Feed body mixin.
+  module FeedBody
+
+    # Content types to add an enclosure for.
+    EnclosureContentTypes = %w{
+      application/pdf
+      audio/mpeg
+      image/gif
+      image/jpeg
+      image/png
+      }
+
+    def initialize(config, req, feed_url, urls)
+      @config, @req, @feed_url, @urls = config, req, feed_url, urls
+      @updated = nil
+    end
+
+    # Yield body for Rack.
+    def each; yield build; end
+
+    # Build feed title.
+    def feed_title
+      result = "#{config['page_title']}"
+      req['q'] ? "#{result} /#{req['q']}" : result
+    end
+
+    attr_reader :config
+    attr_reader :req
+    attr_reader :feed_url
+    attr_reader :urls
+    attr_reader :updated
+  end
+
+end

data/lib/murlsh/img_store.rb

@@ -1,7 +1,5 @@
-require 'cgi'
 require 'digest/md5'
 require 'open-uri'
-require 'uri'
 
 require 'RMagick'
 
@@ -9,66 +7,67 @@ require 'murlsh'
 
 module Murlsh
 
-  # Fetch images from urls and store them locally.
+  # Store images from various sources in asset storage.
+  #
+  # Storage is determined by store_asset plugins.
   class ImgStore
 
-    # Fetch images from urls and store them locally.
-    # Options:
-    # * :user_agent - user agent to send with http requests
-    def initialize(storage_dir, options={})
-      @storage_dir = storage_dir
-      @user_agent = options[:user_agent]
+    # Store images from various sources in asset storage.
+    #
+    # Storage is determined by store_asset plugins.
+    def initialize(config)
+      @config = config
     end
 
     # Build headers to send with request.
     def headers
       result = {}
-      result['User-Agent'] = @user_agent  if @user_agent
+      result['User-Agent'] = config['user_agent']  if config['user_agent']
       result
     end
 
-    # Fetch an image from a url and store it locally.
-    #
-    # The filename will be the md5sum of the contents plus the correct
-    # extension.
+    # Fetch an image from a url and store it in asset storage.
     #
     # If a block is given the Magick::ImageList created will be yielded
     # before storage.
+    #
+    # Returns image url.
     def store_url(url, &block)
-      open(url, headers) { |fin| store_img_data fin.read, &block }
+      open(url, headers) { |fin| store_img_data(fin.read, &block) }
     end
 
-    # Accept a blob of image data and store it locally.
-    #
-    # The filename will be the md5sum of the contents plus the correct
-    # extension.
+    # Store a blob of image data in asset storage.
     #
     # If a block is given the Magick::ImageList created will be yielded
     # before storage.
+    #
+    # Returns image url.
     def store_img_data(img_data, &block)
       img = Magick::ImageList.new.from_blob(img_data)
       yield img if block_given?
       store_img img
     end
 
-    # Accept a Magick::ImageList and store it locally.
+    # Store a Magick::ImageList in asset storage.
     #
-    # The filename will be the md5sum of the contents plus the correct
-    # extension.
+    # Returns image url.
     def store_img(img)
       img.extend(Murlsh::ImageList)  unless img.is_a?(Murlsh::ImageList)
       img_data = img.to_blob
       md5 = Digest::MD5.hexdigest(img_data)
 
-      local_file = "#{md5}#{img.preferred_extension}"
-      local_path = File.join(storage_dir, local_file)
-      unless File.exists?(local_path)
-        Murlsh::openlock(local_path, 'w') { |fout| fout.write img_data }
+      name = "img/thumb/#{md5}#{img.preferred_extension}"
+
+      Murlsh::Plugin.hooks('store_asset') do |p|
+        # run until one returns something
+        if url = p.run(name, img_data, config)
+          return url
+        end
       end
-      local_file
+      nil
     end
 
-    attr_reader :storage_dir
+    attr_reader :config
   end
 
 end

data/lib/murlsh/install.rb

@@ -18,6 +18,7 @@ module Murlsh
         Rakefile
         config.ru
         config.yaml
+        db/
         plugins/
         public/
       }.map { |x| File.join(MurlshRoot, x) }, dest_dir, :verbose => true)

data/lib/murlsh/json_body.rb

@@ -4,7 +4,6 @@ module Murlsh
 
   # Recent urls json response builder.
   class JsonBody
-    include Murlsh::BuildMd5
 
     def initialize(config, req, result_set)
       @config, @req, @result_set = config, req, result_set
@@ -19,6 +18,10 @@ module Murlsh
         @body
       else
         urls = @result_set.results.map do |mu|
+          Murlsh::Plugin.hooks('url_display_pre') do |p|
+            p.run mu, @req, @config
+          end
+
           h = mu.attributes
 
           h['title'] = mu.title_stripped
@@ -26,7 +29,7 @@ module Murlsh
           # add site root url to relative thumbnail urls
           if h['thumbnail_url'] and
             not URI(h['thumbnail_url']).scheme.to_s.downcase[/https?/]
-            h['thumbnail_url'] = URI.join(@config['root_url'],
+            h['thumbnail_url'] = URI.join(@config.fetch('root_url'),
               h['thumbnail_url']).to_s
           end
 

data/lib/murlsh/json_server.rb

@@ -5,8 +5,6 @@ module Murlsh
   # Serve most recent urls in json and jsonp.
   class JsonServer
 
-    include Murlsh::HeadFromGet
-
     def initialize(config); @config = config; end
 
     # Respond to a GET request. Return json of recent urls or jsonp if
@@ -14,26 +12,24 @@ module Murlsh
     def get(req)
       conditions = Murlsh::SearchConditions.new(req['q']).conditions
       page = 1
-      per_page = @config.fetch('num_posts_feed', 25)
+      per_page = config.fetch('num_posts_feed', 25)
 
       result_set = Murlsh::UrlResultSet.new(conditions, page, per_page)
 
-      resp = Rack::Response.new
-
       if req['callback']
-        resp['Content-Type'] = 'application/javascript'
-        resp.body = Murlsh::JsonpBody.new(@config, req, result_set)
+        content_type = 'application/javascript'
+        body = Murlsh::JsonpBody.new(config, req, result_set)
       else
-        resp['Content-Type'] = 'application/json'
-        resp.body = Murlsh::JsonBody.new(@config, req, result_set)
+        content_type = 'application/json'
+        body = Murlsh::JsonBody.new(config, req, result_set)
       end
 
-      resp['Cache-Control'] = 'must-revalidate, max-age=0'
-      resp['ETag'] = "\"#{resp.body.md5}\""
-
-      resp
+      Rack::Response.new body, 200,
+        'Cache-Control' => 'must-revalidate, max-age=0',
+        'Content-Type' => content_type
     end
 
+    attr_reader :config
   end
 
 end

data/lib/murlsh/m3u_body.rb

@@ -0,0 +1,28 @@
+module Murlsh
+
+  # m3u builder.
+  class M3uBody
+    include Murlsh::FeedBody
+
+    # m3u builder.
+    def build
+      if defined?(@body)
+        @body
+      else
+        result = "# #{feed_url}\r\n\r\n"
+        urls.each do |mu|
+          Murlsh::Plugin.hooks('url_display_pre') do |p|
+            p.run mu, req, config
+          end
+
+          result << "#{mu.url}\r\n"
+          @updated = @updated ? [@updated, mu.time].max : mu.time
+        end
+
+        @body = result
+      end
+    end
+
+  end
+
+end

data/lib/murlsh/m3u_server.rb

@@ -0,0 +1,50 @@
+require 'uri'
+
+require 'rack'
+
+require 'murlsh'
+
+module Murlsh
+
+  # Serve m3u file of audio urls.
+  class M3uServer
+
+    AudioContentTypes = %w{
+      application/ogg
+      audio/mpeg
+      audio/ogg
+      }
+
+    def initialize(config); @config = config; end
+
+    # Respond to a GET request for m3u file.
+    def get(req)
+      conditions = ['content_type IN (?)', AudioContentTypes]
+      search_conditions = Murlsh::SearchConditions.new(req['q']).conditions
+      unless search_conditions.empty?
+        conditions[0] << " AND (#{search_conditions[0]})"
+        conditions.push(*search_conditions[1..-1])
+      end
+
+      page = 1
+      per_page = config.fetch('num_posts_feed', 25)
+
+      result_set = Murlsh::UrlResultSet.new(conditions, page, per_page)
+      urls = result_set.results
+
+      feed_url = URI.join(config.fetch('root_url'), 'm3u.m3u')
+      body = Murlsh::M3uBody.new(config, req, feed_url, urls)
+
+      resp = Rack::Response.new(body, 200,
+        'Cache-Control' => 'must-revalidate, max-age=0',
+        'Content-Type' => 'audio/x-mpegurl')
+      if u = body.updated
+        resp['Last-Modified'] = u.httpdate
+      end
+      resp
+    end
+
+    attr_reader :config
+  end
+
+end

data/lib/murlsh/markup.rb

@@ -70,7 +70,7 @@ module Murlsh
       if options[:id]
         if options[:label]
           label_suffix = options[:label_suffix] || ':'
-          label("#{options[:label]}#{label_suffix}", :for => options[:id])
+          label "#{options[:label]}#{label_suffix}", :for => options[:id]
         end
         options[:name] ||= options[:id]
       end

data/lib/murlsh/plugin.rb

@@ -9,8 +9,13 @@ module Murlsh
   #   run arguments (config hash)
   # * avatar - called to get an avatar url from an email md5 sum
   #   run arguments (avatar url, url, config hash)
+  # * store_asset - store an asset somewhere where it can be loaded by url
+  #   run arguments (name, data, config hash)
   # * url_display_add - called to display additional information after urls
   #   run arguments (markup builder, url, config hash)
+  # * url_display_pre - called to modify a url on-the-fly before display, does
+  #   not change database
+  #   run arguments (url, rack request, config hash)
   class Plugin
 
     # Called when a plugin class inherits from this class (the way plugins

data/lib/murlsh/podcast_server.rb

@@ -0,0 +1,44 @@
+require 'uri'
+
+require 'rack'
+
+require 'murlsh'
+
+module Murlsh
+
+  # Serve podcast RSS feed.
+  class PodcastServer
+
+    def initialize(config); @config = config; end
+
+    # Respond to a GET request for podcast RSS feed.
+    def get(req)
+      conditions = ['content_type = ?', 'audio/mpeg']
+      search_conditions = Murlsh::SearchConditions.new(req['q']).conditions
+      unless search_conditions.empty?
+        conditions[0] << " AND (#{search_conditions[0]})"
+        conditions.push(*search_conditions[1..-1])
+      end
+
+      page = 1
+      per_page = config.fetch('num_posts_feed', 25)
+
+      result_set = Murlsh::UrlResultSet.new(conditions, page, per_page)
+      urls = result_set.results
+
+      feed_url = URI.join(config.fetch('root_url'), 'podcast.rss')
+      body = Murlsh::RssBody.new(config, req, feed_url, urls)
+
+      resp = Rack::Response.new(body, 200,
+        'Cache-Control' => 'must-revalidate, max-age=0',
+        'Content-Type' => 'application/rss+xml')
+      if u = body.updated
+        resp['Last-Modified'] = u.httpdate
+      end
+      resp
+    end
+
+    attr_reader :config
+  end
+
+end

data/lib/murlsh/pop_server.rb

@@ -0,0 +1,78 @@
+require 'net/pop'
+
+require 'active_record'
+require 'json'
+require 'postrank-uri'
+require 'rack'
+require 'rmail'
+
+module Murlsh
+
+  # Pop mail from a pop3 server and add all urls in messages from murlsh users.
+  class PopServer
+
+    def initialize(config); @config = config; end
+
+    def post(req)
+      response_body = []
+
+      if req['secret'] == config['pop_secret']
+        Net::POP3.enable_ssl
+        Net::POP3.start(config.fetch('pop_server'), config.fetch('pop_port'),
+          config.fetch('pop_user'), config.fetch('pop_password')) do |pop|
+          pop.each_mail do |mail|
+            begin
+              response_body << process_mail(mail.pop)
+            rescue Exception
+            ensure
+              mail.delete
+            end
+          end
+          pop.finish
+        end
+      end
+
+      Rack::Response.new response_body.to_json, 200,
+        'Content-Type' => 'application/json'
+    end
+
+    # Authenticate the sender and add all urls extracted from the the email
+    # body.
+    def process_mail(mail)
+      parsed_mail = parse_mail(mail)
+
+      if user = Murlsh::Auth.new(config.fetch('auth_file')).by_email(
+        parsed_mail[:from])
+        parsed_mail[:uris].each do |uri|
+          mu = Murlsh::Url.new do |u|
+            u.url = uri
+            u.email = user[:email]
+            u.name = user[:name]
+            u.time = parsed_mail[:date]
+          end
+
+          # validate before add_pre plugins have run and also after (in save!)
+          raise ActiveRecord::RecordInvalid.new(mu)  unless mu.valid?
+          Murlsh::Plugin.hooks('add_pre') { |p| p.run mu, config }
+          mu.save!
+          Murlsh::Plugin.hooks('add_post') { |p| p.run mu, config }
+        end
+      end
+      parsed_mail
+    end
+
+    # Parse date, from address and urls found in body from an email message.
+    def parse_mail(mail)
+      parsed_mail = RMail::Parser.read(mail.gsub(/\r/, ''))
+
+      {
+        :date => parsed_mail.header.date.utc,
+        :from => parsed_mail.header.from.first.address.downcase,
+        :uris => PostRank::URI.extract(parsed_mail.body),
+      }    
+    end
+
+    attr_reader :config
+  end
+
+end

data/lib/murlsh/random_server.rb

@@ -0,0 +1,41 @@
+require 'rack'
+
+require 'murlsh'
+
+module Murlsh
+
+  # Redirect to a random url from the database.
+  class RandomServer
+
+    def initialize(config); @config = config; end
+
+    # Redirect to a random url from the database optionally matching a query.
+    #
+    # Redirect to root url if no urls match.
+    def get(req)
+      if choice = random_url(Murlsh::SearchConditions.new(req['q']).conditions)
+        url = choice.url
+      else
+        url = config.fetch('root_url')
+      end
+
+      resp = Rack::Response.new("<a href=\"#{url}\">#{url}</a>")
+      resp.redirect(url)
+
+      resp
+    end
+
+    # Select a random url from the database optionally matching a query.
+    #
+    # Return nil if no urls match.
+    def random_url(conditions=[])
+      count = Murlsh::Url.count(:conditions => conditions)
+      if count > 0
+        Murlsh::Url.first(:conditions => conditions, :offset => rand(count))
+      end
+    end
+
+    attr_reader :config
+  end
+
+end