file_worker 0.0.1

data/bin/file_worker

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "bundler"
+Bundler.require
+
+require 'file_worker/cli'
+
+scanner = FileWorker::Cli.build_scanner
+
+scanner.on_error do |file_name, exception|
+  $stderr.puts("Failed to process #{file_name}")
+  $stderr.puts(ex)
+  $stderr.puts(ex.backtrace.join("\n"))
+end
+
+scanner.start

data/lib/file_worker/cli.rb

@@ -0,0 +1,48 @@
+require 'optparse'
+
+module FileWorker
+  class Cli
+    def self.build_scanner
+      options = {
+        :workers => 5
+      }
+
+      OptionParser.new do |opts|
+        opts.banner = "Usage: #{$0} [options]"
+
+        opts.on("-w", "--workers [NUMBER]", "The number of worker threads") do |workers|
+          options[:workers] = workers.to_i
+        end
+
+        opts.on("-o", "--out [DIRECTORY]", "The directory to put the files when they have been processed") do |out|
+          options[:out_directory] = out
+        end
+
+        opts.on("-s", "--sleep [NUMBER]", Float, "The number of seconds to sleep between scanning the in-directory") do |sleep_time|
+          options[:sleep] = sleep_time
+        end
+
+        opts.on("-q", "--queuesize [NUMBER]", "The maximum queue size to keep in memory") do |max_queue_size|
+          options[:max_queue_size] = max_queue_size.to_i
+        end
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+      end.parse!
+
+      if root = ARGV[0]
+        options[:in_directory] = File.expand_path(root, Dir.pwd)
+      else
+        options[:in_directory] = Dir.pwd
+      end
+
+      options[:out_directory] ||= File.expand_path('../done', options[:in_directory])
+
+      puts options.inspect
+
+      DirectoryScanner.new(options)
+    end
+  end
+end

data/lib/file_worker/default_worker.rb

@@ -0,0 +1,11 @@
+module FileWorker
+  class DefaultWorker
+    def initialize(file_name, options)
+      @file_name   = file_name
+    end
+
+    def process
+      puts @file_name
+    end
+  end
+end

data/lib/file_worker/directory_scanner.rb

@@ -0,0 +1,100 @@
+require 'girl_friday'
+require 'fileutils'
+require 'jruby/synchronized'
+require 'pathname'
+
+module FileWorker
+  class DirectoryScanner
+    attr_accessor :worker_class
+    attr_reader :in_path, :done_path, :state
+
+    def initialize(options)
+      @options   = options
+      @in_path   = Pathname.new(@options[:in_directory])
+      @done_path = Pathname.new(@options[:out_directory])
+      @sleep     = @options[:sleep] || 1
+
+      @max_queue_size = @options[:max_queue_size] || 1000
+
+      @worker_class = DefaultWorker
+
+      @state = {}
+      @state.extend JRuby::Synchronized
+      @queue_name = "file_worker"
+
+      @error_handlers = []
+    end
+
+    # This method is called by the worker threads, so remember to keep it thread safe
+    def process(file_name)
+      @state[file_name] = {:time => Time.now, :status => :working}
+
+      begin
+        @worker_class.new(file_name, @options).process
+        FileUtils.mv(file_name, @done_path)
+      rescue Exception => e
+        handle_error(file_name, e)
+      end
+
+      @state.delete(file_name)
+    end
+
+    def queue
+      @queue ||= GirlFriday::WorkQueue.new(@queue_name, :size => 3) do |file_name|
+        process(file_name)
+      end
+    end
+
+    def queue_size
+      queue.status[@queue_name][:backlog]
+    end
+
+    def enqueue(file_name)
+      @state[file_name] = {:time => Time.now, :status => :enqueued}
+
+      queue.push(file_name)
+    end
+
+    def wait_for_empty
+      queue.wait_for_empty
+
+      sleep 0.5
+
+      while queue.status[@queue_name][:busy] != 0
+        sleep 0.5
+      end
+    end
+
+    def scan
+      file_names = Dir.glob(@in_path + '*') - @state.keys
+
+      max_items = @max_queue_size - queue_size
+
+      file_names[0,max_items].each do |file_name|
+        enqueue(file_name)
+      end
+    end
+
+    def start
+      @run = true
+      while @run
+        scan
+        sleep(@sleep)
+      end
+    end
+
+    def stop
+      @run = false
+    end
+
+    def on_error(&block)
+      @error_handlers << block
+    end
+
+    def handle_error(file_name, exception)
+      @error_handlers.each do |error_handler|
+        error_handler.call(file_name, exception)
+      end
+    end
+  end
+end

data/lib/file_worker/s3_upload_worker.rb

@@ -0,0 +1,34 @@
+require "fog"
+
+module FileWorker
+  class S3UploadWorker
+    def initialize(file_name, options)
+      @file_name   = file_name
+      @bucket_name = ENV.fetch("S3_BUCKET")
+      @aws_key_id  = ENV.fetch("AWS_ACCESS_KEY_ID")
+      @aws_secret  = ENV.fetch("AWS_SECRET_ACCESS_KEY")
+    end
+
+    def process
+      bucket.files.create(
+        :key    => File.basename(@file_name),
+        :body   => File.open(@file_name),
+        :public => false
+      )
+    end
+
+    private
+
+    def connection
+      @connection ||= Fog::Storage.new(
+        :provider              => "AWS",
+        :aws_access_key_id     => @aws_key_id,
+        :aws_secret_access_key => @aws_secret
+      )
+    end
+
+    def bucket
+      connection.directories.get(@bucket_name)
+    end
+  end
+end

data/lib/file_worker/version.rb

@@ -0,0 +1,3 @@
+module FileWorker
+  VERSION = "0.0.1"
+end

data/lib/file_worker.rb

@@ -0,0 +1,7 @@
+require "file_worker/version"
+
+module FileWorker
+  autoload :DirectoryScanner, 'file_worker/directory_scanner'
+  autoload :DefaultWorker,    'file_worker/default_worker'
+  autoload :S3UploadWorker,   'file_worker/s3_upload_worker'
+end

data/test/directory_scanner_test.rb

@@ -0,0 +1,126 @@
+require File.expand_path("test_helper", File.dirname(__FILE__))
+require 'jruby/synchronized'
+
+class FailingTestWorker
+  def initialize(file_name, options)
+    @file_name = file_name
+  end
+
+  def process
+    raise @file_name
+  end
+end
+
+describe "the directory scanner" do
+  before do
+    prepare_fixture_files
+    @file_worker = FileWorker::DirectoryScanner.new(
+      :in_directory => @fixture_root + 'in',
+      :out_directory => @fixture_root + 'done',
+      :max_queue_size => 10
+    )
+
+    @errors = []
+    @errors.extend JRuby::Synchronized
+
+    @file_worker.on_error do |file_name, exception|
+      @errors << file_name
+    end
+  end
+
+  it "should enqueue the files in the in directory" do
+    @file_worker.queue.expects(:push).times(5)
+    @file_worker.scan
+    @file_worker.wait_for_empty
+  end
+
+  it "should move the files to the out directory" do
+    @file_worker.scan
+    @file_worker.wait_for_empty
+
+    @file_worker.in_path.children.must_be_empty
+    @file_worker.done_path.children.size.must_equal(5)
+  end
+
+  it "should not enqueue the this same files twice" do
+    @file_worker.queue.expects(:push).times(5)
+    @file_worker.scan
+    @file_worker.wait_for_empty
+    @file_worker.scan
+    @file_worker.wait_for_empty
+  end
+
+  describe "when the worker queue is empty" do
+    before do
+      @file_worker.stubs(:queue_size).returns(0)
+    end
+
+    it "should only enqueue 10 items" do
+      prepare_fixture_files(20)
+      @file_worker.queue.expects(:push).times(10)
+      @file_worker.scan
+      @file_worker.wait_for_empty
+    end
+  end
+
+  describe "when the worker queue has some items in it" do
+    before do
+      @file_worker.stubs(:queue_size).returns(3)
+    end
+
+    it "should not try to over fill the queue" do
+      prepare_fixture_files(20)
+      @file_worker.queue.expects(:push).times(7)
+      @file_worker.scan
+      @file_worker.wait_for_empty
+    end
+  end
+
+  describe "when the worker queue is full" do
+    before do
+      @file_worker.stubs(:queue_size).returns(10)
+    end
+
+    it "should not enqueue anything" do
+      prepare_fixture_files(20)
+      @file_worker.queue.expects(:push).never
+      @file_worker.scan
+      @file_worker.wait_for_empty
+    end
+  end
+
+  describe "when a file is already enqueued" do
+    before do
+      file_name = Dir.glob(@file_worker.in_path + '*').first
+      @file_worker.state[file_name] = {:time => Time.now, :status => :working}
+    end
+
+    it "should not enqueue the file again" do
+      @file_worker.queue.expects(:push).times(4)
+      @file_worker.scan
+      @file_worker.wait_for_empty
+    end
+  end
+
+  describe "when jobs fail" do
+    before do
+      @file_worker.worker_class = FailingTestWorker
+      @file_worker.scan
+      @file_worker.wait_for_empty
+    end
+
+    it "should call the error handler" do
+      @errors.size.must_equal(5)
+    end
+
+    it "should not move the files" do
+      @file_worker.done_path.children.must_be_empty
+      @file_worker.in_path.children.size.must_equal(5)
+    end
+
+    it "should remove the files from state" do
+      @file_worker.state.must_be_empty
+    end
+  end
+
+end

data/test/s3_upload_worker_test.rb

@@ -0,0 +1,35 @@
+require File.expand_path("test_helper", File.dirname(__FILE__))
+
+describe "an s3 upload worker" do
+  before do
+    @file_name = "/path/to/file.name"
+  end
+
+  it "should fail when not provided auth options" do
+    assert_raises(IndexError) { FileWorker::S3UploadWorker.new(@file_name, {}) }
+  end
+
+  describe "instantiated correctly" do
+    before do
+      ENV["S3_BUCKET"]             = "files"
+      ENV["AWS_ACCESS_KEY_ID"]     = "some key"
+      ENV["AWS_SECRET_ACCESS_KEY"] = "other key"
+
+      @worker = FileWorker::S3UploadWorker.new(@file_name, {})
+    end
+
+    describe "process" do
+      it "should attempt to transfer to the location" do
+        file   = stub()
+        File.expects(:open).with(@file_name).returns(file)
+
+        files  = stub()
+        bucket = stub(:files => files)
+        files.expects(:create).with(:key => File.basename(@file_name), :body => file, :public => false)
+
+        @worker.expects(:bucket).returns(bucket)
+        @worker.process
+      end
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,37 @@
+$testing = true
+
+require "rubygems"
+require "bundler"
+Bundler.require
+
+require "minitest/autorun"
+require "mocha"
+require "pathname"
+require "fileutils"
+
+MiniTest::Spec.class_eval do
+  before do
+    @fixture_root = Pathname.new(File.dirname(__FILE__)) + "../tmp/fixtures"
+    FileUtils.mkdir_p(@fixture_root + "in")
+    FileUtils.mkdir_p(@fixture_root + "done")
+    clean_fixture_dirs
+  end
+
+  def clean_fixture_dirs
+    FileUtils.rm_rf(Dir.glob(@fixture_root + "in/*"))
+    FileUtils.rm_rf(Dir.glob(@fixture_root + "done/*"))
+  end
+
+  def prepare_fixture_files(in_files = 5, done_files = 0)
+    create_fixture_files(@fixture_root + "in", in_files)
+    create_fixture_files(@fixture_root + "done", done_files)
+  end
+
+  def create_fixture_files(path, number)
+    number.times do |n|
+      File.open(path + n.to_s, "w") do |file|
+        file.write(n.to_s)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification 
+name: file_worker
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+  - Mick Staugaard
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-01-26 00:00:00 Z
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: rake
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :development
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: minitest
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :development
+    version_requirements: *id002
+  - !ruby/object:Gem::Dependency 
+    name: mocha
+    prerelease: false
+    requirement: &id003 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :development
+    version_requirements: *id003
+  - !ruby/object:Gem::Dependency 
+    name: jruby-openssl
+    prerelease: false
+    requirement: &id004 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id004
+  - !ruby/object:Gem::Dependency 
+    name: girl_friday
+    prerelease: false
+    requirement: &id005 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id005
+  - !ruby/object:Gem::Dependency 
+    name: fog
+    prerelease: false
+    requirement: &id006 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ">="
+          - !ruby/object:Gem::Version 
+            version: "0"
+    type: :runtime
+    version_requirements: *id006
+description: If you have files that you some how need to process, file_worker is your friend.
+email: 
+  - mick@staugaard.com
+executables: 
+  - file_worker
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+  - lib/file_worker.rb
+  - lib/file_worker/cli.rb
+  - lib/file_worker/default_worker.rb
+  - lib/file_worker/directory_scanner.rb
+  - lib/file_worker/s3_upload_worker.rb
+  - lib/file_worker/version.rb
+  - README.md
+  - test/directory_scanner_test.rb
+  - test/s3_upload_worker_test.rb
+  - test/test_helper.rb
+  - bin/file_worker
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: A multi-threaded worker that takes files as input
+test_files: 
+  - test/directory_scanner_test.rb
+  - test/s3_upload_worker_test.rb
+  - test/test_helper.rb