ungulate 0.1.4 → 0.2.0

data/lib/ungulate/job.rb

@@ -1,162 +1,42 @@
-require 'rubygems'
-require 'right_aws'
-require 'RMagick'
-require 'mime/types'
 require 'yaml'
-require 'active_support/core_ext'
-require 'curb'
 
 module Ungulate
   class Job
-    attr_accessor :bucket, :key, :notification_url, :queue, :versions
-
-    @@blobs_from_urls = {}
-    cattr_accessor :blobs_from_urls
-
-    def self.s3
-      @s3 ||=
-        RightAws::S3.new(ENV['AMAZON_ACCESS_KEY_ID'],
-                         ENV['AMAZON_SECRET_ACCESS_KEY'])
+    def initialize(options = {})
+      @logger = options[:logger] || ::Logger.new($stdout)
+      @blob_processor = options[:blob_processor]
+      @storage = options[:storage]
+      @http = options[:http]
     end
 
-    def self.sqs
-      @sqs ||= 
-        RightAws::SqsGen2.new(ENV['AMAZON_ACCESS_KEY_ID'],
-                              ENV['AMAZON_SECRET_ACCESS_KEY'])
-    end
+    def process(encoded_job)
+      @attributes = YAML.load(encoded_job)
+      bucket = @storage.bucket(@attributes[:bucket], :listener => self)
+      versions = @attributes[:versions]
+      blob = bucket.retrieve(@attributes[:key])
 
-    def self.pop(queue_name)
-      job = new
-      job.queue = sqs.queue queue_name
-      message = job.queue.pop
-      attributes = YAML.load message.to_s
-      job.attributes = attributes if attributes
-      job
+      @blob_processor.process(
+        :blob => blob, :versions => versions,
+        :bucket => bucket, :original_key => @attributes[:key]
+      )
     end
 
-    def initialize
-      @logger = Ungulate::Server.logger
-      self.versions = []
-    end
+    def storage_complete(version)
+      stored_versions << version
 
-    def attributes=(options)
-      self.bucket = Job.s3.bucket(options[:bucket])
-      self.key = options[:key]
-      self.notification_url = options[:notification_url]
-      self.versions = options[:versions]
-    end
-
-    def processed_versions
-      @processed_versions ||=
-        versions.map do |name, instruction|
-          [name, processed_image(source_image, instruction)]
-        end
-    end
-
-    def blob_from_url(url)
-      Job.blobs_from_urls[url] ||=
-        begin
-          @logger.info "Grabbing blob from URL #{url}"
-          Curl::Easy.http_get(url).body_str
-        end
-    end
-
-    def magick_image_from_url(url)
-      Magick::Image.from_blob(blob_from_url(url)).first
-    end
-
-    def instruction_args(args)
-      args.map do |arg|
-        if arg.is_a?(Symbol)
-          "Magick::#{arg.to_s.classify}".constantize
-        elsif arg.respond_to?(:match) && arg.match(/^http/)
-          magick_image_from_url(arg)
-        else
-          arg
-        end
-      end
-    end
-
-    def image_from_instruction(original, instruction)
-      method, *args = instruction
-      send_args = instruction_args(args)
-
-      @logger.info "Performing #{method} with #{args.join(', ')}"
-      original.send(method, *send_args).tap do |new_image|
-        original.destroy!
-        send_args.select {|arg| arg.is_a?(Magick::Image)}.each(&:destroy!)
+      if @attributes[:notification_url] && stored_versions == versions_to_process
+        @http.put @attributes[:notification_url]
       end
     end
 
-    def image_from_instruction_chain(original, chain)
-      if chain.one?
-        image_from_instruction(original, chain.first)
-      else
-        image_from_instruction_chain(
-          image_from_instruction(original, chain.shift),
-          chain
-        )
-      end
-    end
-
-    def processed_image(original, instruction)
-      if instruction.first.respond_to?(:entries)
-        image_from_instruction_chain(original, instruction)
-      else
-        image_from_instruction(original, instruction)
-      end
-    end
-
-    def source_image
-      Magick::Image.from_blob(source).first
-    end
-
-    def source
-      if @source
-        @source
-      else
-        @logger.info "Grabbing source image #{key}"
-        @source = bucket.get key
-      end
-    end
-
-    def process
-      return false if processed_versions.empty?
-
-      processed_versions.each do |version, image|
-        version_key = version_key version
-        @logger.info "Storing #{version} @ #{version_key}"
-        bucket.put(
-          version_key, 
-          image.to_blob, 
-          {},
-          'public-read',
-          {
-            'Content-Type' => MIME::Types.type_for(image.format).to_s,
-            # expire in about one month: refactor to grab from job description
-            'Cache-Control' => 'max-age=2629743',
-          }
-        )
-        image.destroy!
-      end
-
-      send_notification
-
-      true
-    end
-
-    def send_notification
-      return false if notification_url.blank?
+    protected
 
-      @logger.info "Sending notification to #{notification_url}"
-      Curl::Easy.http_put(notification_url, '')
+    def stored_versions
+      @stored_versions ||= Set.new
     end
 
-    def version_key(version)
-      dirname = File.dirname(key)
-      extname = File.extname(key)
-      basename = File.basename(key, extname)
-      "#{dirname}/#{basename}_#{version}#{extname}".sub(/^\.\//, '')
+    def versions_to_process
+      @versions_to_process ||= Set.new @attributes[:versions].keys
     end
   end
 end

data/lib/ungulate/rmagick_version_creator.rb

@@ -0,0 +1,77 @@
+require 'rmagick'
+require 'active_support/core_ext/string'
+require 'mime/types'
+
+module Ungulate
+  class RmagickVersionCreator
+    def initialize(options = {})
+      @logger = options[:logger] || ::Logger.new($stdout)
+      @http = options[:http]
+    end
+
+    def create(blob, instructions)
+      image = processed_image(magick_image_from_blob(blob), instructions)
+      {
+        :blob => image.to_blob,
+        :content_type => MIME::Types.type_for(image.format).to_s
+      }
+    end
+
+    protected
+
+    def blob_from_url(url)
+      @blobs_from_urls ||= {}
+      @blobs_from_urls[url] ||= @http.get_body(url)
+    end
+
+    def magick_image_from_url(url)
+      Magick::Image.from_blob(blob_from_url(url)).first
+    end
+
+    def instruction_args(args)
+      args.map do |arg|
+        if arg.is_a?(Symbol)
+          "Magick::#{arg.to_s.classify}".constantize
+        elsif arg.respond_to?(:match) && arg.match(/^http/)
+          magick_image_from_url(arg)
+        else
+          arg
+        end
+      end
+    end
+
+    def image_from_instruction(original, instruction)
+      method, *args = instruction
+      send_args = instruction_args(args)
+
+      @logger.info "Performing #{method} with #{args.join(', ')}"
+      original.send(method, *send_args).tap do |new_image|
+        original.destroy!
+        send_args.select {|arg| arg.is_a?(Magick::Image)}.each(&:destroy!)
+      end
+    end
+
+    def image_from_instruction_chain(original, chain)
+      if chain.one?
+        image_from_instruction(original, chain.first)
+      else
+        image_from_instruction_chain(
+          image_from_instruction(original, chain.shift),
+          chain
+        )
+      end
+    end
+
+    def processed_image(original, instruction)
+      if instruction.first.respond_to?(:entries)
+        image_from_instruction_chain(original, instruction)
+      else
+        image_from_instruction(original, instruction)
+      end
+    end
+
+    def magick_image_from_blob(blob)
+      Magick::Image.from_blob(blob).first
+    end
+  end
+end

data/lib/ungulate/s3_storage.rb

@@ -0,0 +1,41 @@
+module Ungulate
+  class S3Storage
+    def initialize(options)
+      @s3 = RightAws::S3.new *options.values_at(:access_key_id, :secret_access_key)
+    end
+
+    def bucket(name, options = {})
+      S3Bucket.new(@s3, name, options)
+    end
+  end
+
+  class S3Bucket
+    def initialize(s3, name, options = {})
+      @bucket = s3.bucket(name)
+      @listener = options[:listener]
+      @logger = options[:logger] || ::Logger.new($stdout)
+    end
+
+    def store(key, value, options = {})
+      @logger.info "Storing #{key} with size #{value.size}, content-type #{options[:content_type]}"
+
+      @bucket.put(key, value, {}, 'public-read', {
+        'Content-Type' => options[:content_type],
+        'Cache-Control' => 'max-age=2629743'
+      })
+
+      if @listener && options[:version]
+        @listener.storage_complete(options[:version])
+      end
+    end
+
+    def retrieve(key)
+      @logger.info "Retrieving #{key}"
+      @bucket.get(key)
+    end
+
+    def clear
+      @bucket.clear
+    end
+  end
+end

data/lib/ungulate/server.rb

@@ -1,15 +1,35 @@
 require 'logger'
-require 'ungulate/job'
 
 module Ungulate
-  module Server
-    def self.logger
-      @logger ||= ::Logger.new STDOUT
+  class Server
+    def initialize(options = {})
+      @logger = options[:logger] || ::Logger.new($stdout)
+      @job_processor = options[:job_processor]
+      @queue = options[:queue]
     end
 
-    def self.run(queue_name)
-      logger.info "Checking for job on #{queue_name}"
-      Job.pop(queue_name).process
+    class << self
+      def config
+        Ungulate.configuration
+      end
+
+      def run
+        new(
+          :job_processor => config.job_processor.call,
+          :queue => config.queue.call
+        ).run
+      end
+    end
+
+    def run
+      @logger.info "Checking for job on #{@queue.name}"
+      message = @queue.receive
+
+      if message
+        success = @job_processor.process(message.to_s)
+        message.delete
+        success
+      end
     end
   end
 end

data/lib/ungulate/sqs_message_queue.rb

@@ -0,0 +1,38 @@
+require 'right_aws'
+
+module Ungulate
+  class SqsMessageQueue
+    def initialize(name, options)
+      if name.blank?
+        raise Ungulate::MissingConfiguration,
+          "queue_name must be set in config"
+      end
+
+      sqs = RightAws::SqsGen2.new(
+        options[:access_key_id], options[:secret_access_key],
+        :server => options[:server]
+      )
+      @queue = sqs.queue name
+    end
+
+    def name
+      @queue.name
+    end
+
+    def clear
+      @queue.clear
+    end
+
+    def push(message)
+      @queue.push(message)
+    end
+
+    def receive
+      @queue.receive
+    end
+
+    def size
+      @queue.size
+    end
+  end
+end

data/spec/lib/ungulate/blob_processor_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+require 'ungulate/blob_processor'
+
+module Ungulate
+  describe BlobProcessor do
+    subject { BlobProcessor.new(:version_creator => creator) }
+
+    let(:creator) { double 'version creator' }
+    let(:bucket) { double 'bucket' }
+
+    it "processes a blob into several versions and sends them to storage" do
+      blob = 'asdf'
+
+      large = [
+        [ :resize_to_fill, 400, 300 ],
+        [ :composite, 'http://some.image/url.jpg', :center_gravity, :soft_light_composite_op ],
+      ]
+      medium = [ :resize_to_fit, 300, 200 ]
+      thumbnail = [ :resize_to_fit, 40, 20 ]
+
+      versions = {
+        :large => large,
+        :medium => medium,
+        :thumbnail => thumbnail
+      }
+
+
+      creator.should_receive(:create).with(blob, large).
+        and_return(:blob => 'largeblob', :content_type => 'image/png')
+
+      creator.should_receive(:create).with(blob, medium).
+        and_return(:blob => 'mediumblob', :content_type => 'image/jpeg')
+
+      creator.should_receive(:create).with(blob, thumbnail).
+        and_return(:blob => 'thumbnailblob', :content_type => 'application/xml')
+
+      bucket.should_receive(:store).
+        with('some/file_large.jpg', 'largeblob', :version => :large,
+             :content_type => 'image/png')
+
+      bucket.should_receive(:store).
+        with('some/file_medium.jpg', 'mediumblob', :version => :medium,
+             :content_type => 'image/jpeg')
+
+      bucket.should_receive(:store).
+        with('some/file_thumbnail.jpg', 'thumbnailblob', :version => :thumbnail,
+             :content_type => 'application/xml')
+
+      subject.process(
+        :blob => blob, :versions => versions,
+        :original_key => 'some/file.jpg', :bucket => bucket
+      )
+    end
+
+    context "when key has no leading path" do
+      it "converts the key properly" do
+        creator.stub(:create).and_return({})
+
+        bucket.should_receive(:store).with('file_large.jpg', anything, anything)
+
+        subject.process(
+          :blob => 'asdf', :versions => { :large => [] },
+          :original_key => 'file.jpg', :bucket => bucket
+        )
+      end
+    end
+  end
+end