sqs_image_processor 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c4e61adf1c5d48f2606df7964d4ed1a0ba765089
+  data.tar.gz: 852021f5d287a612a5b8e24fc28c27c37ed511e7
+SHA512:
+  metadata.gz: 36d80e0c3b592d91bd82a75594f0c1ee12ae92a0fdc84efb0c14db71017b7d9c604a3207bb8b7e09ee03862d29a11154dd8eec60651436dd196c1a7b01403506
+  data.tar.gz: ad3f55a9626ca737834dc0d773623bfc446a92ff40611af0dd6e3fbb3294c4c5e977151950b5859328efbf66e8fe111656a5d40c6b5b3c4bb549eb66342e46c9

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.rvmrc
+/sqs_image_processor_config.yml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sqs_image_processor.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Neil Turner
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# Sqs Image Processor
+
+Queued image processing which utilizes AWS::S3 for storage and AWS::SQS for queueing.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sqs_image_processor'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sqs_image_processor
+
+## Usage
+
+Configuration is via a simple YAML file. Run the following command to generate an example in your current working directory.
+    
+    $ sqs_image_processor generate_config
+
+Edit the settings as required, and when you're ready to start the processing use the following command.
+
+    $ sqs_image_processor start
+
+Add the -d flag to run the process in the background.
+
+SQS Image Processor will search your current working directory for a configuration file. If you'd like to specify a different location use the -c option.
+
+    $ sqs_image_processor start -c path/to/config.yml
+
+To stop the processing use the 'stop' command.
+
+    $ sqs_image_processor stop
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/sqs_image_processor/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/sqs_image_processor

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'sqs_image_processor'
+
+# Entry-point to the application.
+SqsImageProcessor::Manager.start(ARGV)

data/lib/sqs_image_processor/config.rb

@@ -0,0 +1,10 @@
+require 'yaml'
+require 'recursive-open-struct'
+
+module SqsImageProcessor
+  class Config
+    def self.load( config_path )
+      RecursiveOpenStruct.new( YAML.load_file(config_path) )
+    end
+  end
+end

data/lib/sqs_image_processor/generators/config.rb

@@ -0,0 +1,19 @@
+require 'fileutils'
+
+module SqsImageProcessor
+  module Generators
+    def self.config_generator
+      current_directory = File.dirname(__FILE__)
+      working_directory = File.expand_path(Dir.pwd)
+      target_path = File.join(working_directory, 'sqs_image_processor_config.yml')
+
+      if File.exists?(target_path)
+        puts "Config file already exists at this location."
+      else
+        template = File.join(current_directory, 'templates', 'sqs_image_processor_config.yml')
+        FileUtils.cp(template, target_path)
+        puts "Successfully generated config file in current working directory."
+      end
+    end
+  end
+end

data/lib/sqs_image_processor/generators/templates/sqs_image_processor_config.yml

@@ -0,0 +1,21 @@
+aws:
+  sqs:
+    access_key_id: YOUR-AWS-ACCESS-KEY
+    secret_access_key: YOUR-AWS-SECRET-ACCESS-KEY
+    region: eu-west-1
+    queue_name: image-queue
+  s3:
+    access_key_id: YOUR-AWS-ACCESS-KEY
+    secret_access_key: YOUR-AWS-SECRET-ACCESS-KEY
+    region: eu-west-1
+    bucket: your-image-bucket
+versions:
+  default:
+    width: 1000
+    height: 1000
+  thumbnail:
+    width: 300
+    height: 300
+  web:
+    width: 600
+    height: 600

data/lib/sqs_image_processor/manager.rb

@@ -0,0 +1,62 @@
+require "thor"
+
+# :stopdoc:
+# SQS Image Processor offers a minimalist interface to make using it
+# as easy as we can. The command line interface is implemented using
+# Thor but this could easily be standard ruby if we want to strip out
+# dependencies and streamline the gem.
+# :startdoc:
+module SqsImageProcessor 
+
+  # Main interface to the application. Provides 4 commands to the user.
+  #
+  # - start
+  # - stop
+  # - status
+  # - generate_config
+  class Manager < Thor
+
+    desc "start", "Start the SQS Image Processor."
+    method_option :c, :type => :string, :default => File.join(Dir.pwd, 'sqs_image_processor_config.yml'), :required => false
+    method_option :d, :type => :boolean, :default => false, :required => false
+    def start
+      config_path = options[:c]
+      daemonize = options[:d]
+
+      if !File.exists?(config_path)
+        puts "Error: Config file not found."
+      elsif SqsImageProcessor::ProcessManager.is_running?
+        puts "Error: An instance of SqsImageProcessor is already running."
+      else
+        Process.daemon if daemonize
+        SqsImageProcessor::ProcessManager.generate_pid_file
+        config = SqsImageProcessor::Config.load( config_path )
+        puts "Loaded config file at #{config_path}."
+        puts "Starting SQS Image Processor."
+        SqsImageProcessor::Worker.start( config )
+      end
+    end
+
+    desc "stop", "Stop the SQS Image Processor."
+
+    def stop
+      SqsImageProcessor::ProcessManager.kill
+    end
+
+    desc "status", "Check whether SQS Image Processor is running."
+
+    def status
+      if SqsImageProcessor::ProcessManager.is_running?
+        puts "Running with PID #{SqsImageProcessor::ProcessManager.get_pid}."
+      else
+        puts "Not running."
+      end
+    end
+
+    desc "generate_config", "Generate an example SQS Image Processor configuration file in the current wokring directory."
+
+    def generate_config
+      SqsImageProcessor::Generators.config_generator
+    end
+  end
+end

data/lib/sqs_image_processor/process_manager.rb

@@ -0,0 +1,41 @@
+module SqsImageProcessor
+  module ProcessManager
+    def self.pid_is_running?( pid )
+      begin
+        Process.getpgid( pid.to_i )
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+    def self.kill
+      if self.is_running?
+        pid = File.open("/tmp/sqs_image_processor_manager.pid", 'a+') {|f| f.read }.to_i
+        Process.kill("KILL", pid)
+        puts "Stopped instance of SqsImageProcessor."
+      else
+        puts "Error: SqsImageProcessor isn't running."
+      end
+    end
+
+    def self.is_running?
+      pid = File.open("/tmp/sqs_image_processor_manager.pid", 'a+') {|f| f.read }.to_i
+      if pid == 0 || !self.pid_is_running?(pid)
+        false
+      else
+        true
+      end
+    end
+
+    def self.generate_pid_file
+      File.open("/tmp/sqs_image_processor_manager.pid", 'w') {|f|
+        f.write(Process.pid)
+      }
+    end
+
+    def self.get_pid
+      pid = File.open("/tmp/sqs_image_processor_manager.pid", 'a+') {|f| f.read }.to_i
+    end
+  end
+end

data/lib/sqs_image_processor/version.rb

@@ -0,0 +1,3 @@
+module SqsImageProcessor
+  VERSION = "0.1.0"
+end

data/lib/sqs_image_processor/worker.rb

@@ -0,0 +1,69 @@
+require 'aws-sdk-core'
+require 'uri'
+
+module SqsImageProcessor
+  module Worker
+    def self.start( config )
+      continue_processing = true
+      
+      sqs_client = Aws::SQS::Client.new(
+        credentials: Aws::Credentials.new(config.aws.sqs.access_key_id, config.aws.sqs.secret_access_key),
+        region: config.aws.sqs.region
+      )
+      s3_client = Aws::S3::Client.new(
+        credentials: Aws::Credentials.new(config.aws.s3.access_key_id, config.aws.s3.secret_access_key),
+        region: config.aws.sqs.region
+      )
+      queue = sqs_client.get_queue_url({queue_name: config.aws.sqs.queue_name})
+
+      while continue_processing
+        begin
+          resp = sqs_client.receive_message( {queue_url: queue.queue_url} )
+          if resp.messages != nil
+            url = "http://#{config.aws.s3.bucket}.s3.amazonaws.com/#{resp.messages[0].body}"
+            puts "Converting #{url}"
+            system("wget -O /tmp/#{File.basename(resp.messages[0].body)} #{url} --no-cache > /dev/null")
+            config.versions.to_h.each do |k, v|
+              version_name = k
+              width = v['width']
+              height = v['height']
+              filename = File.basename(resp.messages[0].body)
+              version_filename = "#{filename.chomp(File.extname(filename))}_#{version_name}#{File.extname(filename)}"
+              version_path = "#{resp.messages[0].body.chomp(filename)}#{version_filename}}"
+              system("gm convert /tmp/#{filename} -resize '#{width}x#{height}' +profile \"*\" /tmp/#{version_filename} > /dev/null")
+
+              # Upload to S3
+              s3_resp = s3_client.put_object(
+                acl: "public-read",
+                body: File.open("/tmp/#{version_filename}"),
+                bucket: config.aws.s3.bucket,
+                key: version_path
+              )
+
+              puts 'SAVED: ' + config.aws.s3.bucket + '/' + version_path
+
+            end
+            sqs_client.delete_message(
+              queue_url: queue.queue_url,
+              receipt_handle: resp.messages[0].receipt_handle
+            )
+          end
+        rescue
+          # Immediately return this item to the queue for processing.
+          begin
+            if resp && resp.messages != nil
+              sqs_client.change_message_visibility(
+                queue_url: queue.queue_url,
+                receipt_handle: resp.messages[0].receipt_handle,
+                visibility_timeout: 0
+              )
+            end
+          rescue
+            # Just continue anyway
+          end
+        end
+        sleep 0.1
+      end
+    end
+  end
+end

data/lib/sqs_image_processor.rb

@@ -0,0 +1,9 @@
+require "sqs_image_processor/version"
+require "sqs_image_processor/manager"
+require "sqs_image_processor/config"
+require "sqs_image_processor/process_manager"
+require "sqs_image_processor/worker"
+require "sqs_image_processor/generators/config"
+
+module SqsImageProcessor
+end

data/sqs_image_processor.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sqs_image_processor/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sqs_image_processor"
+  spec.version       = SqsImageProcessor::VERSION
+  spec.authors       = ["Neil Turner"]
+  spec.email         = ["neil@neilturner.me"]
+  spec.summary       = %q{Write a short summary. Required.}
+  spec.description   = %q{Write a longer description. Optional.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  #spec.executables   = [spec.files.grep(%r{^bin/}) { |f| File.basename(f) }]
+  spec.executables   = ["sqs_image_processor"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor", "~> 0.19.1"
+  spec.add_dependency "recursive-open-struct", "~> 0.5.0"
+  spec.add_dependency "aws-sdk-core"  #, "~> 2.0"
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: sqs_image_processor
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Neil Turner
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: recursive-open-struct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Write a longer description. Optional.
+email:
+- neil@neilturner.me
+executables:
+- sqs_image_processor
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/sqs_image_processor
+- lib/sqs_image_processor.rb
+- lib/sqs_image_processor/config.rb
+- lib/sqs_image_processor/generators/config.rb
+- lib/sqs_image_processor/generators/templates/sqs_image_processor_config.yml
+- lib/sqs_image_processor/manager.rb
+- lib/sqs_image_processor/process_manager.rb
+- lib/sqs_image_processor/version.rb
+- lib/sqs_image_processor/worker.rb
+- sqs_image_processor.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Write a short summary. Required.
+test_files: []