airbrush 0.0.2

data/History.txt

@@ -0,0 +1,4 @@
+== 0.0.1 2008-01-22
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,55 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/airbrush
+bin/airbrush-example-client
+config/hoe.rb
+config/requirements.rb
+lib/airbrush.rb
+lib/airbrush/client.rb
+lib/airbrush/core_ext/get_args.rb
+lib/airbrush/core_ext/timestamped_logger.rb
+lib/airbrush/handler.rb
+lib/airbrush/listeners/listener.rb
+lib/airbrush/listeners/memcache.rb
+lib/airbrush/listeners/socket.rb
+lib/airbrush/listeners/webservice.rb
+lib/airbrush/processors/image/image_processor.rb
+lib/airbrush/processors/image/profiles/cmyk-profile.icc
+lib/airbrush/processors/image/profiles/srgb-profile.icc
+lib/airbrush/processors/image/rmagick.rb
+lib/airbrush/processors/processor.rb
+lib/airbrush/publishers/http.rb
+lib/airbrush/publishers/memcache.rb
+lib/airbrush/publishers/publisher.rb
+lib/airbrush/server.rb
+lib/airbrush/version.rb
+log/debug.log
+script/destroy
+script/generate
+script/txt2html
+setup.rb
+spec/airbrush/client_spec.rb
+spec/airbrush/core_ext/get_args_spec.rb
+spec/airbrush/handler_spec.rb
+spec/airbrush/listeners/listener_spec.rb
+spec/airbrush/listeners/memcache_spec.rb
+spec/airbrush/processors/image/image_processor_spec.rb
+spec/airbrush/processors/image/rmagick_spec.rb
+spec/airbrush/processors/processor_spec.rb
+spec/airbrush/publishers/memcache_spec.rb
+spec/airbrush/publishers/publisher_spec.rb
+spec/airbrush/server_spec.rb
+spec/airbrush_spec.rb
+spec/spec.opts
+spec/spec_helper.rb
+tasks/deployment.rake
+tasks/environment.rake
+tasks/rspec.rake
+tasks/website.rake
+website/index.html
+website/index.txt
+website/javascripts/rounded_corners_lite.inc.js
+website/stylesheets/screen.css
+website/template.rhtml

data/README.txt

@@ -0,0 +1,39 @@
+= AIRBRUSH
+
+== DESCRIPTION
+
+Airbrush is a distributed processing host for performing arbitrary tasks on a particular server.
+Currently it supports using MemCache as the transport queue to communicate between an airbrush
+server issuing a command and the server processing it.
+
+Due to the distributed nature of MemCache, multiple Airbrush servers can be started across
+many hosts, and each will monitor the MemCache incoming queue for results independently and
+atomically.
+
+Remote jobs are performed via a 'processor' installed as part of Airbrush, and is currently
+configured in code in the server.rb source file (look for the handler definition/configuration). A
+future enhancemnt could support automatic processor registration and/or a plugin architecture.
+
+Actual communication between client and server via MemCache is done via a hash, that includes
+a :command value, optional :args value and a uniqie :id value. The :id value is used to uniquely
+identify this job, :command names the actual command to run on the remote server, and :params
+is a hash of parameters that are passed to the command being executed.
+
+Several conventions are used, the name of the command needs to match a method name on the
+processor. The key names in the params hash need to match parameter names to the method being
+invoked (and will be automatically assigned). The :id value is used to name a queue that the
+client can poll for return values from the remote command, if required.
+
+== USAGE
+
+To use Airbrush, you will need a starling server installed and running:
+
+    $> starling -v -P /tmp/starling.pid -q /tmp/queue
+
+and at least one Airbrush instance running, including your processor:
+
+    $> airbrush -v
+
+To send a remote job to the Airbrush server, use the Airbrush::Client module located
+in airbrush/client.rb. An example is airbrush-example-client, which sends an image to the
+remote server to be turned into a set of previews.

data/Rakefile

@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/bin/airbrush

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+#
+#  Created on 2008-1-22.
+#  Copyright (c) 2008. All rights reserved.
+
+begin
+  require 'rubygems'
+rescue LoadError
+  # no rubygems to load, so we fail silently
+end
+
+require 'optparse'
+
+OPTIONS = {
+  :memcache => '127.0.0.1:22122',
+  :frequency => 2,
+  :verbose => false
+}
+MANDATORY_OPTIONS = %w( )
+
+parser = OptionParser.new do |opts|
+  opts.banner = <<BANNER
+Airbrush is a dedicated job queue based image processor. Commands are added to the queue via memcache, and results posted back
+via memcache. This portion of airbrush is the server component, please see the examples are of the gem for how to post and
+receive results using an example client.
+
+Usage: #{File.basename($0)} [options]
+
+Options are:
+BANNER
+  opts.separator ""
+  opts.on("-m", "--memcache=HOST", String,
+          "The address of the memcache host to connect to",
+          "Default: #{OPTIONS[:memcache]}") { |OPTIONS[:memcache]| }
+  opts.on("-f", "--frequency=SECONDS", String,
+          "How often the remote memcache server is polled for incoming jobs, units of seconds",
+          "Default: #{OPTIONS[:frequency]}") { |OPTIONS[:frequency]| }
+  opts.on("-l", "--log=file", String,
+          "Specify where to send logging output",
+          "Default: stdout") { |OPTIONS[:log_target]| }
+  opts.on("-v", "--verbose",
+          "Verbose output") { |OPTIONS[:verbose]| }
+  opts.on("-h", "--help",
+          "Show this help message.") { puts opts; exit }
+  opts.parse!(ARGV)
+
+  if MANDATORY_OPTIONS && MANDATORY_OPTIONS.find { |option| OPTIONS[option.to_sym].nil? }
+    puts opts; exit
+  end
+end
+
+# all good, lets go
+
+require 'airbrush'
+require 'daemons'
+
+# Become a daemon
+#Daemons.daemonize
+
+server = Airbrush::Server.new(OPTIONS)
+server.start

data/bin/airbrush-example-client

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+#
+#  Created on 2008-1-22.
+#  Copyright (c) 2008. All rights reserved.
+
+begin
+  require 'rubygems'
+rescue LoadError
+  # no rubygems to load, so we fail silently
+end
+
+require 'optparse'
+
+OPTIONS = {
+  :memcache => '127.0.0.1:22122',
+  :verbose => false
+}
+MANDATORY_OPTIONS = %w( image output )
+
+parser = OptionParser.new do |opts|
+  opts.banner = <<BANNER
+Example client for accessing the resize operation on a remote airbrush server, queued via memcache.
+
+Usage: #{File.basename($0)} [options]
+
+Options are:
+BANNER
+  opts.separator ""
+  opts.on("-m", "--memcache=HOST", String,
+          "The address of the memcache host to connect to",
+          "Default: #{OPTIONS[:memcache]}") { |OPTIONS[:memcache]| }
+  opts.on("-i", "--image=FILENAME", String,
+          "Source input image") { |OPTIONS[:image]| }
+  opts.on("-o", "--output=FILENAME", String,
+          "Target output image") { |OPTIONS[:output]| }
+  opts.on("-v", "--verbose",
+          "Verbose output") { |OPTIONS[:verbose]| }
+  opts.on("-h", "--help",
+          "Show this help message.") { puts opts; exit }
+  opts.parse!(ARGV)
+
+  if MANDATORY_OPTIONS && MANDATORY_OPTIONS.find { |option| OPTIONS[option.to_sym].nil? }
+    puts opts; exit
+  end
+end
+
+raise "Source image does not exist: #{OPTIONS[:image]}" unless File.exists?(OPTIONS[:image])
+
+# all good, lets go!
+
+require 'airbrush'
+
+client = Airbrush::Client.new(OPTIONS[:memcache])
+puts "Sending #{OPTIONS[:image]} for preview processing"
+results = client.process('generate-previews', :previews, :image => File.read(OPTIONS[:image]), :filename => OPTIONS[:image], :sizes => {:small => [300], :large => [600]})
+results.each do |k,v|
+  next if k == :original
+  File.open("#{OPTIONS[:output]}-#{k}.jpg", 'w') { |f| f << v[:image] }
+  puts "Saved preview #{OPTIONS[:output]}-#{k}.jpg (#{v[:width]}x#{v[:height]})" if OPTIONS[:verbose]
+end

data/config/hoe.rb

@@ -0,0 +1,71 @@
+require 'airbrush/version'
+
+AUTHOR = 'FIXME full name'  # can also be an array of Authors
+EMAIL = "FIXME email"
+DESCRIPTION = "description of gem"
+GEM_NAME = 'airbrush' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'airbrush' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "unknown"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Airbrush::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'airbrush documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  p.extra_deps = [ [ 'starling', '>= 0.9.3' ], [ 'activesupport', '>= 2.0.2' ], [ 'ruby2ruby', '>= 1.1.8' ], [ 'daemons', '>= 1.0.9' ] ]     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+hoe.rsync_args = '-av --delete --ignore-errors'

data/config/requirements.rb

@@ -0,0 +1,17 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
+
+require 'airbrush'

data/lib/airbrush/client.rb

@@ -0,0 +1,53 @@
+require 'starling'
+require 'timeout'
+
+module Airbrush
+  include Timeout
+  class AirbrushProcessingError < StandardError; end
+
+  class Client
+    DEFAULT_INCOMING_QUEUE   = 'airbrush_incoming_queue'
+    DEFAULT_RESPONSE_TIMEOUT = 2.minutes
+    DEFAULT_QUEUE_VALIDITY   = 0 # 10.minutes for the moment
+
+    attr_reader :host, :incoming_queue, :response_timeout, :queue_validity
+
+    def initialize(host, incoming_queue = DEFAULT_INCOMING_QUEUE, response_timeout = DEFAULT_RESPONSE_TIMEOUT, queue_validity = DEFAULT_QUEUE_VALIDITY)
+      @host = host
+      @server = Starling.new(@host)
+      @incoming_queue = incoming_queue
+      @response_timeout = response_timeout
+      @queue_validity = queue_validity.to_i
+    end
+
+    def process(id, command, args = {})
+      raise 'No job id specified' unless id
+      raise 'No command specified' unless command
+      raise "Invalid arguments #{args}" unless args.is_a? Hash
+
+      send_and_receive(id, command, args)
+    end
+
+    private
+
+      def send_and_receive(id, command, args)
+        @server.set(@incoming_queue, { :id => id, :command => command, :args => args }, @queue_validity, false)
+        queue = unique_name(id)
+        Timeout::timeout(@response_timeout) do
+          response = @server.get(queue)
+          raise AirbrushProcessingError, format_error(response) if response.include? :exception
+          return response
+        end
+      end
+
+      # REVISIT: share implementation with server?
+      def unique_name(id)
+        id.to_s
+      end
+
+      def format_error(response)
+        "#{response[:exception]}: #{response[:message]}"
+      end
+
+  end
+end

data/lib/airbrush/core_ext/get_args.rb

@@ -0,0 +1,94 @@
+require 'ruby2ruby'
+require 'parse_tree'
+
+class ParseTreeArray < Array #:nodoc:
+  def self.translate(*args)
+    sexp = ::ParseTree.translate(*args)
+    # ParseTree.translate returns [nil] if called on an inherited method, so walk
+    # up the inheritance chain to find the class that the method was defined in
+    unless sexp.first
+      klass = args.first.ancestors.detect do |klass|
+        klass.public_instance_methods(false).include?(args.last.to_s)
+      end
+      sexp = ::ParseTree.translate(klass, args.last) if klass
+    end
+    self.new(sexp)
+  end
+
+  def deep_array_node(type = nil)
+    each do |node|
+      return ParseTreeArray.new(node) if node.is_a?(Array) && (!type || node[0] == type)
+      next unless node.is_a?(Array)
+      return ParseTreeArray.new(node).deep_array_node(type)
+    end
+    nil
+  end
+
+  def arg_nodes
+    self[1..-1].inject([]) do |sum,item|
+      sum << [item] unless item.is_a?(Array)
+      sum
+    end
+  end
+
+  def get_args
+    if arg_node = deep_array_node(:args)
+      # method defined with def keyword
+      args = arg_node.arg_nodes
+      default_node = arg_node.deep_array_node(:block)
+      return args unless default_node
+    else
+      # assuming method defined with Module#define_method
+      return []
+    end
+
+    # if it was defined with def, and we found the default_node,
+    # that should bring us back to regularly scheduled programming..
+
+    lasgns = default_node[1..-1]
+    lasgns.each do |asgn|
+      args.assoc(asgn[1]) << eval(RubyToRuby.new.process(asgn[2]))
+    end
+    args
+  end
+
+end
+
+# Used in mapping controller arguments to the params hash.
+# NOTE: You must have the 'ruby2ruby' gem installed for this to work.
+#
+# ==== Examples
+#   # In PostsController
+#   def show(id)  #=> id is the same as params[:id]
+module GetArgs
+
+  # ==== Returns
+  # Array:: Method arguments and their default values.
+  #
+  # ==== Examples
+  #   class Example
+  #     def hello(one,two="two",three)
+  #     end
+  #
+  #     def goodbye
+  #     end
+  #   end
+  #
+  #   Example.instance_method(:hello).get_args
+  #     #=> [[:one], [:two, "two"], [:three, "three"]]
+  #   Example.instance_method(:goodbye).get_args  #=> nil
+  def get_args
+    klass, meth = self.to_s.split(/ /).to_a[1][0..-2].split("#")
+    # Remove stupidity for #<Method: Class(Object)#foo>
+    klass = $` if klass =~ /\(/
+    ParseTreeArray.translate(Object.const_get(klass), meth).get_args
+  end
+end
+
+class UnboundMethod #:nodoc:
+  include GetArgs
+end
+
+class Method  #:nodoc:
+  include GetArgs
+end

data/lib/airbrush/core_ext/timestamped_logger.rb

@@ -0,0 +1,10 @@
+module ActiveSupport
+  class BufferedLogger
+
+    def add_with_timestamps(severity, message = nil, progname = nil, &block)
+      add_without_timestamps(severity, "#{Time.now}: #{message}", progname, &block)
+    end
+
+    alias_method_chain :add, :timestamps
+  end
+end

data/lib/airbrush/handler.rb

@@ -0,0 +1,18 @@
+module Airbrush  
+  class Handler
+    attr_reader :processor, :publisher
+    
+    def initialize(processor, publisher)
+      raise ArgumentError, 'no processor specified' unless processor
+      raise ArgumentError, 'no publisher specified' unless publisher
+      
+      @processor = processor
+      @publisher = publisher
+    end
+
+    def process(id, command, args)
+      @publisher.publish id, @processor.dispatch(command, args)
+    end
+
+  end
+end

data/lib/airbrush/listeners/listener.rb

@@ -0,0 +1,45 @@
+module Airbrush
+  module Listeners
+    class Listener
+      attr_accessor :handler
+      
+      def start
+        raise 'implementations provide concrete listener functionality'
+      end
+      
+      protected
+      
+        def process(op)
+          raise 'No operation specified' unless op
+          
+          unless valid?(op)
+            log.error "Received invalid job #{op}"
+            return
+          end
+          
+          log.debug "Processing #{op[:id]}"
+          start = Time.now
+          
+          begin
+            @handler.process op[:id], op[:command], op[:args]
+          rescue Exception => e
+            log.error 'Received error during handler'
+            log.error e
+          ensure
+            log.debug "Processed #{op[:id]}: #{Time.now - start} seconds processing time"
+          end
+        end
+        
+      private
+        
+        def valid?(op)
+          return false unless op.is_a? Hash
+          return false unless op[:id]
+          return false unless op[:command]
+          return false unless op[:args]
+          true
+        end
+        
+    end
+  end
+end

data/lib/airbrush/listeners/memcache.rb

@@ -0,0 +1,53 @@
+require 'memcache'
+
+module Airbrush
+  module Listeners
+    class Memcache < Listener
+      DEFAULT_POLL_FREQUENCY = 2 # seconds
+      DEFAULT_INCOMING_QUEUE = 'airbrush_incoming_queue'
+
+      attr_reader :host, :poll_frequency
+
+      def initialize(host, frequency = DEFAULT_POLL_FREQUENCY)
+        @host = host
+        @poll_frequency = frequency
+        catch_signals(:int)
+      end
+
+      def start
+        @running = true
+        @starling = MemCache.new(@host)
+
+        log.debug 'Accepting incoming jobs'
+
+        loop do
+          poll do |operation|
+            process operation
+          end
+
+          break unless @running
+          sleep @poll_frequency
+        end
+
+        @starling.reset
+      end
+
+      private
+
+        def poll
+          operation = @starling.get(DEFAULT_INCOMING_QUEUE)
+          yield operation if operation and block_given?
+        end
+
+        def catch_signals(*signals)
+          signals.each do |signal|
+            sig = signal.to_s.upcase
+            Signal.trap(sig) do
+              @running = false
+              log.debug "Intercepted SIG#{sig}, exiting"
+            end
+          end
+        end
+    end
+  end
+end

data/lib/airbrush/processors/image/image_processor.rb

@@ -0,0 +1,31 @@
+module Airbrush
+  module Processors
+    module Image
+      class ImageProcessor < Processor
+        class_inheritable_accessor :before_filters, :after_filters
+
+        def self.before_filter(*symbols)
+          self.before_filters = symbols
+        end
+
+        def self.after_filter(*symbols)
+          self.after_filters = symbols
+        end
+        
+        def dispatch(command, args)
+          self.before_filters.each { |filter| filter_dispatch(filter, args) } if self.before_filters
+          rv = super command, args
+          self.after_filters.each { |filter| filter_dispatch(filter, args) } if self.after_filters
+          rv
+        end
+        
+        def filter_dispatch(command, args)
+          raise "Unknown processor operation #{command} (#{args.inspect unless args.blank?})" unless self.respond_to? command
+          params = assign(command, args)
+          self.send command, *params
+        end
+        
+      end
+    end
+  end
+end