file-processing-job 0.1.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 cj2
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,57 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "file-processing-job"
+    gem.summary = %Q{Light-weight client / server implementation for processing disk based records}
+    gem.description = %Q{file-processing-job allows you to distribute the processing load of large files to clients across the network. It is a thin wrapper on top of the EventMachine library.}
+    gem.email = "cjgrimes@gmail.com"
+    gem.homepage = "http://github.com/cj2/file-processing-job"
+    gem.authors = ["cj2"]
+    gem.files = FileList["[A-Z]*.*", "{examples,lib,features,spec}/**/*", "Rakefile", "cucumber.yml"]
+    gem.add_development_dependency "aasm", "~> 2.1"
+    gem.add_development_dependency "directory_watcher", "~> 1.3"
+    gem.add_development_dependency "eventmachine", "~> 0.12"
+    
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "file-processing-job #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/examples/test_client.rb

@@ -0,0 +1,10 @@
+require 'file-processing-job'
+
+class FileProcessor
+  def receive_file data
+    puts "received file:"
+    p data
+  end
+end
+
+FileProcessingJob::connect('127.0.0.1', 11222, FileProcessor)

data/examples/test_server.rb

@@ -0,0 +1,6 @@
+require 'file-processing-job'
+
+FileProcessingJob::start_server() {|config|
+  config.inbox_directory = './data/inbox'
+  config.processed_directory = './data/processed'
+}

data/lib/file-processing-job.rb

@@ -0,0 +1,23 @@
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'eventmachine'
+require 'aasm'
+require 'directory_watcher'
+
+module FileProcessingJob
+  def self.logger
+    @@logger ||= begin
+      if (const_defined?('Rails'))
+        Rails.logger
+      else
+        require 'logger'
+        Dir.mkdir('log') unless Dir.exists?('log')
+        Logger.new('log/fileprocessingjob.log')
+      end
+    end
+  end
+end
+
+require 'fpj/server'
+require 'fpj/client'
+

data/lib/fpj/client.rb

@@ -0,0 +1,94 @@
+# add special parsers - parse JSON files, XML files into nokogiri, etc
+# go to eventmachine forum - see if anyone has already done this
+
+module FileProcessingJob
+  def self.connect(host='127.0.0.1',port=11222,callback)
+    EM::run {
+      client_connection = Client::Connection
+      client_connection.callback = callback
+      EventMachine::connect host, port, client_connection
+    }
+  end
+
+  module Client
+    class Config
+      attr_accessor :parse_file
+    end
+
+    class Connection < EM::Connection
+      include AASM
+
+      class << self
+        attr_accessor :callback
+      end
+
+      attr_accessor :filename, :data
+
+      aasm_state :requesting_task
+      aasm_state :downloading, :enter => :download_file
+      aasm_state :processing, :enter => :process_file
+      aasm_state :complete, :exit => :request_task
+
+      aasm_event :filename_received do
+        transitions :to => :downloading, :from => [:requesting_task]
+      end
+
+      aasm_event :file_received do
+        transitions :to => :processing, :from => [:downloading]
+      end
+
+      aasm_event :file_processed do
+        transitions :to => :complete, :from => [:processing]
+      end
+
+      aasm_event :next_task do
+        transitions :to => :requesting_task, :from => [:complete]
+      end
+
+      def logger
+        FileProcessingJob.logger
+      end
+      
+      
+      def request_task
+        self.filename = nil
+        self.data = nil
+        send_data "next"
+      end
+
+      def download_file
+        self.filename = self.data
+        send_data "get #{self.filename}"
+      end
+
+      def process_file
+        begin
+          self.class.callback.new.send(:receive_file, self.data)
+          send_data "processed #{self.filename}"
+        rescue
+          logger.error("Error processing #{self.filename}. The file can be found on the server in the 'error' directory")
+          logger.error($!)
+          send_data "error #{self.filename}"
+        end
+      end
+
+      def post_init
+        request_task
+      end
+
+      def receive_data(data)
+        self.data = data
+        case aasm_current_state
+          when :requesting_task
+            filename_received
+          when :downloading
+            file_received
+          when :processing
+            file_processed
+            next_task
+          end
+      end
+    end
+  end
+end
+

data/lib/fpj/server.rb

@@ -0,0 +1,186 @@
+# add special parsers - parse JSON files, XML files into nokogiri, etc
+# go to eventmachine forum - see if anyone has already done this
+
+require 'fileutils'
+
+module FileProcessingJob
+  # FileProcessingJob::start_server creates an event machine server
+  # on the specified IP address and port. The optional configuration
+  # object passed to start_server can be used to point FileProcessingJob
+  # to the directories used to manage the files. 
+  #
+  # By default, FileProcessingJob will look in the following subdirectories
+  # to manage files. If the directories do not exist they will be created
+  # automatically at runtime.
+  #
+  # Files placed in the "inbox" directory will automatically detected and
+  # sent to a worker for processing.
+  #
+  # ./data/inbox
+  # ./data/processing
+  # ./data/processed
+  #
+  # === Logging
+  #
+  # By default the server and clients will use the Rails logging facility
+  # if deployed within a Rails application. Otherwise a logger will be
+  # created that logs to ./log/file_processing_job.log
+  #
+  # === Usage example
+  #
+  #  The server monitors the inbox directory for new files and dispatches
+  #  the contents of each file to the next available worker. The worker
+  #  then processed the file or raises an exception and the server handles
+  #  moving the file to either the processed or error directory. Any number
+  #  of workers can exist anywhere on the network. Note that by default
+  #  the server and client bind to 127.0.0.1
+  #
+  #  require 'rubygems'
+  #  require 'file-processing-job'
+  #
+  #  Server:
+  #
+  #  FileProcessingJob::start_server() {|config|
+  #    config.inbox_directory = './data/inbox'
+  #    config.processing_directory = './data/processing'
+  #    config.processed_directory = './data/processed'
+  #  }
+  #
+  #  Client:
+  #
+  #  class FileProcessor
+  #    def receive_file data
+  #      puts "received file data: #{data}"
+  #      # do something interesting here
+  #    end
+  #  end
+  # 
+  #  FileProcessingJob::connect('127.0.0.1', 11222, FileProcessor)
+  
+  def self.server_config
+  end
+  
+  def self.stop_server
+    
+  end
+  
+  def self.start_server(host='127.0.0.1',port=11222,&block)
+    Server.config(&block) if block
+
+    # create the directories if they do not exist
+    Server.create_directories
+
+    # watch inbox directory
+    dw = DirectoryWatcher.new Server::config.inbox_directory, :glob => '*'
+    dw.add_observer {|*args| 
+      args.each {|event| 
+        if (event.type == :added)
+          filename = event.path.sub(Server::config.inbox_directory, '').sub(/^\//, '')
+          logger.debug "#{filename} discovered in inbox"
+          Server::Connection.push(filename) 
+        end
+      }
+    }
+    dw.start
+
+    # run the server
+    EM::run {
+      EM::start_server host, port, Server::Connection
+      FileProcessingJob.logger.info "server started: #{host}:#{port}"
+    }
+    dw.stop
+  end
+  
+  module Server
+    def self.create_directories
+      [config.inbox_directory, config.processing_directory, config.processed_directory, config.error_directory].each do |dir|
+        FileUtils.mkpath(dir) unless Dir.exists?(dir)
+      end
+    end
+    
+    def self.config(&block)
+      @@config ||= Config.new()
+      block.call(@@config) if block
+      @@config
+    end
+    
+    class Config
+      attr_writer :inbox_directory, :processed_directory, :processing_directory, :error_directory
+      
+      def inbox_directory
+        @inbox_directory || './data/inbox'
+      end
+      
+      def processed_directory
+        @processed_directory || './data/processed'
+      end
+      
+      def processing_directory
+        @processing_directory || './data/processing'
+      end
+      
+      def error_directory
+        @error_directory || './data/error'
+      end
+    end
+
+    class Connection < EM::Connection
+      @@q = EM::Queue.new()
+      
+      def self.push(message)
+        @@q.push(message)
+      end
+
+      def config 
+        @config ||= FileProcessingJob::Server.config
+      end
+      
+      def logger
+        FileProcessingJob.logger
+      end
+      
+      # inbound client connection
+      def receive_data data
+        data = data.strip
+        case (data)
+        when "next"
+          @@q.pop {|filename| send_data filename }
+        when /^get (.*)$/
+          filename = $1
+          streamer = EventMachine::FileStreamer.new(self, File.join(self.config.inbox_directory, filename))
+          streamer.callback{
+            FileUtils.move(File.join(self.config.inbox_directory, filename), File.join(self.config.processing_directory, filename))
+          }
+          # streamer.error {
+          #   logger.error "Unable to stream file #{filename} to client. The file can be found in the 'error' directory"
+          #   FileUtils.move(File.join(self.config.inbox_directory, filename), File.join(self.config.error_directory, filename))
+          # }
+        when /^processed (.*)$/
+          filename = $1
+          if (File.exists?(File.join(self.config.processed_directory, filename)))
+            logger.error "file #{filename} already exists in processed directory"
+          else
+            FileUtils.move(File.join(self.config.processing_directory, filename), File.join(self.config.processed_directory, filename))
+            logger.debug "finished processing #{filename}"
+          end
+          send_data "ack"
+        when /^error (.*)$/
+          filename = $1
+          FileUtils.move(File.join(self.config.processing_directory, filename), File.join(self.config.error_directory, filename))
+          logger.error "Unable to stream file #{filename} to client. The file can be found in the 'error' directory"
+        when "close"
+          close_connection
+        end
+      end
+  
+      def bind
+        logger.info "client connected to the server"
+      end
+
+      def unbind
+        logger.info "client disconnected from the server"
+      end
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
+$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')))
+
+require 'rubygems'
+require 'aasm'
+require 'file-processing-job'
+require 'rspec'
+require 'rspec/autorun'
+
+Rspec.configure do |config|
+  
+end

data/spec/unit/client_spec.rb

@@ -0,0 +1,54 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+
+class ErrorRaisingProcessor 
+  def receive_file data
+    raise "unknown error"
+  end
+end
+
+class SuccessfulProcessor 
+  def receive_file data
+  end
+end
+
+
+describe FileProcessingJob::Client do
+  describe :prcoess_file do
+    before do
+      EventMachine.stub!(:send_data)
+      @connection = FileProcessingJob::Client::Connection.new(11222)
+      @filename = 'thefile.dat'
+      @connection.filename = @filename
+    end
+
+    describe "success" do
+      before do
+        @handler = SuccessfulProcessor
+        FileProcessingJob::Client::Connection.callback = @handler
+      end
+      
+      it "should send back the success code" do
+        @connection.should_receive(:send_data).with("processed #{@filename}")
+        @connection.process_file
+      end
+    end
+    
+    describe "exception raised" do
+      before do
+        @handler = ErrorRaisingProcessor
+        FileProcessingJob::Client::Connection.callback = @handler
+      end
+      
+      it "should trap and log the exception" do
+        FileProcessingJob.logger.should_receive(:error).twice # once for the human friendly message, once with the stack dump
+        lambda { @connection.process_file }.should_not raise_error
+      end
+
+      it "should send back an error message" do
+        @connection.should_receive(:send_data).with("error #{@filename}")
+        @connection.process_file
+      end
+    end
+
+  end
+end

data/spec/unit/rails_support_spec.rb

@@ -0,0 +1,15 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+
+class Rails
+  def self.logger
+    "rails_logger"
+  end
+end
+
+describe FileProcessingJob do
+  describe "logging" do
+    it "should use the Rails logger" do
+      FileProcessingJob.logger.should eq('rails_logger')
+    end
+  end
+end

data/spec/unit/server_config_spec.rb

@@ -0,0 +1,41 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+
+describe FileProcessingJob::Server do
+  describe :config do
+    describe "defaults" do
+      it "should return ./data/inbox for the inbox directory" do
+        FileProcessingJob::Server::config.inbox_directory.should == './data/inbox'
+      end
+
+      it "should return ./data/processing for the processing directory" do
+        FileProcessingJob::Server::config.processing_directory.should == './data/processing'
+      end
+
+      it "should return ./data/processed for the processed directory" do
+        FileProcessingJob::Server::config.processed_directory.should == './data/processed'
+      end
+    end
+    
+    describe "specified" do
+      before do
+        FileProcessingJob::Server::config do |config|
+          config.inbox_directory = './inbox'
+          config.processing_directory = './processing'
+          config.processed_directory = './processed'
+        end
+      end
+      
+      it "should return ./data/processing for the processing directory" do
+        FileProcessingJob::Server::config.inbox_directory.should == './inbox'
+      end
+
+      it "should return ./data/processing for the processing directory" do
+        FileProcessingJob::Server::config.processing_directory.should == './processing'
+      end
+
+      it "should return ./data/processing for the processing directory" do
+        FileProcessingJob::Server::config.processed_directory.should == './processed'
+      end
+    end
+  end
+end

data/spec/unit/server_spec.rb

@@ -0,0 +1,48 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
+
+describe FileProcessingJob::Server do
+  def mock_streamer
+    o = String.new
+    o.stub!(:callback).and_return nil
+    o.stub!(:error).and_return nil
+    o
+  end
+
+  describe :receive_data do
+    before do
+      @connection = FileProcessingJob::Server::Connection.new(11222)
+      @filename = 'thefile.dat'
+      @config = FileProcessingJob::Server.config
+      @qualified_filename = File.join(@config.inbox_directory, @filename)
+    end
+
+    describe "get" do
+      it "should attempt to stream the file" do
+        EventMachine::FileStreamer.should_receive(:new).with(@connection, @qualified_filename).and_return(mock_streamer)
+        @connection.receive_data("get #{@filename}")
+      end
+    end
+
+    describe "error" do
+      it "should move the file to the error directory" do
+        FileUtils.should_receive(:move).with(File.join(@config.processing_directory, @filename), File.join(@config.error_directory, @filename))
+        @connection.logger.should_receive(:error)
+        @connection.receive_data("error #{@filename}")
+      end
+    end
+  end
+
+  describe :create_directories do
+    it "should create the directories if they do not exist" do
+      Dir.should_receive(:exists?).exactly(4).times.with(an_instance_of(String)).and_return(false)
+      FileUtils.should_receive(:path).exactly(4).times.with(an_instance_of(String))
+      FileProcessingJob::Server.create_directories
+    end
+    
+    it "should not attempt to create the directories if they exist" do
+      Dir.should_receive(:exists?).exactly(4).times.with(an_instance_of(String)).and_return(true)
+      FileUtils.should_not_receive(:mkpath)
+      FileProcessingJob::Server.create_directories
+    end
+  end
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification 
+name: file-processing-job
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- cj2
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-09-09 00:00:00 -06:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: aasm
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 1
+        version: "2.1"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: directory_watcher
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        version: "1.3"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 12
+        version: "0.12"
+  type: :development
+  version_requirements: *id003
+description: file-processing-job allows you to distribute the processing load of large files to clients across the network. It is a thin wrapper on top of the EventMachine library.
+email: cjgrimes@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- examples/test_client.rb
+- examples/test_server.rb
+- lib/file-processing-job.rb
+- lib/fpj/client.rb
+- lib/fpj/server.rb
+- spec/spec_helper.rb
+- spec/unit/client_spec.rb
+- spec/unit/rails_support_spec.rb
+- spec/unit/server_config_spec.rb
+- spec/unit/server_spec.rb
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/cj2/file-processing-job
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Light-weight client / server implementation for processing disk based records
+test_files: 
+- spec/spec_helper.rb
+- spec/unit/client_spec.rb
+- spec/unit/rails_support_spec.rb
+- spec/unit/server_config_spec.rb
+- spec/unit/server_spec.rb
+- examples/test_client.rb
+- examples/test_server.rb