file-processing-job 0.1.1 → 0.1.2

data/Rakefile

@@ -11,7 +11,6 @@ begin
     gem.homepage = "http://github.com/cj2/file-processing-job"
     gem.authors = ["cj2"]
     gem.files = FileList["[A-Z]*.*", "{examples,lib,features,spec}/**/*", "Rakefile", "cucumber.yml"]
-    gem.add_development_dependency "aasm", "~> 2.1"
     gem.add_development_dependency "directory_watcher", "~> 1.3"
     gem.add_development_dependency "eventmachine", "~> 0.12"
     

data/examples/test_client.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'file-processing-job'
 
 class FileProcessor

data/examples/test_server.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'file-processing-job'
 
 FileProcessingJob::start_server() {|config|

data/lib/file-processing-job.rb

@@ -1,10 +1,10 @@
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
 
 require 'eventmachine'
-require 'aasm'
 require 'directory_watcher'
 
 module FileProcessingJob
+  
   def self.logger
     @@logger ||= begin
       if (const_defined?('Rails'))
@@ -18,6 +18,7 @@ module FileProcessingJob
   end
 end
 
+require 'fpj/connection'
 require 'fpj/server'
 require 'fpj/client'
 

data/lib/fpj/client.rb

@@ -3,11 +3,17 @@
 
 module FileProcessingJob
   def self.connect(host='127.0.0.1',port=11222,callback)
-    EM::run {
+    @@running = true
+    while (@@running)
       client_connection = Client::Connection
       client_connection.callback = callback
-      EventMachine::connect host, port, client_connection
-    }
+      EM::run { EventMachine::connect host, port, client_connection }
+    end
+  end
+
+  def self.disconnect
+    @@running = false
+    EventMachine::stop_event_loop
   end
 
   module Client
@@ -15,60 +21,31 @@ module FileProcessingJob
       attr_accessor :parse_file
     end
 
-    class Connection < EM::Connection
-      include AASM
-
+    class Connection < FileProcessingJob::Connection
       class << self
         attr_accessor :callback
       end
 
-      attr_accessor :filename, :data
-
-      aasm_state :requesting_task
-      aasm_state :downloading, :enter => :download_file
-      aasm_state :processing, :enter => :process_file
-      aasm_state :complete, :exit => :request_task
-
-      aasm_event :filename_received do
-        transitions :to => :downloading, :from => [:requesting_task]
-      end
-
-      aasm_event :file_received do
-        transitions :to => :processing, :from => [:downloading]
-      end
-
-      aasm_event :file_processed do
-        transitions :to => :complete, :from => [:processing]
-      end
-
-      aasm_event :next_task do
-        transitions :to => :requesting_task, :from => [:complete]
-      end
-
       def logger
         FileProcessingJob.logger
       end
-      
-      
+
       def request_task
-        self.filename = nil
-        self.data = nil
-        send_data "next"
+        send_message('next')
       end
 
-      def download_file
-        self.filename = self.data
-        send_data "get #{self.filename}"
+      def error(filename)
+        send_message "error #{filename}"
       end
 
-      def process_file
+      def process(filename, contents)
         begin
-          self.class.callback.new.send(:receive_file, self.filename, self.data)
-          send_data "processed #{self.filename}"
+          self.class.callback.new.send(:receive_file, filename, contents)
+          send_message "processed #{filename}"
         rescue
-          logger.error("Error processing #{self.filename}. The file can be found on the server in the 'error' directory")
+          logger.error("Error processing #{filename}. The file can be found on the server in the 'error' directory")
           logger.error($!)
-          send_data "error #{self.filename}"
+          error(filename)
         end
       end
 
@@ -76,17 +53,21 @@ module FileProcessingJob
         request_task
       end
 
-      def receive_data(data)
-        self.data = data
-        case aasm_current_state
-          when :requesting_task
-            filename_received
-          when :downloading
-            file_received
-          when :processing
-            file_processed
-            next_task
-          end
+      def unbind
+        EventMachine::stop_event_loop
+      end
+
+      def receive_task(data)
+        if (data =~ /^filename: (.*)/)
+          @filename = $1
+        else
+          @contents = data
+        end
+
+        if (@filename && @contents)
+          logger.info "Processing #{@filename}"
+          process(@filename, @contents)
+        end
       end
     end
   end

data/lib/fpj/connection.rb

@@ -0,0 +1,25 @@
+module FileProcessingJob
+  class Connection < EM::Connection
+    TOKEN = "_-|fpj|-_"
+
+    def initialize(*args)
+      super
+      @recv_buffer = BufferedTokenizer.new(TOKEN)
+      # set_comm_inactivity_timeout(120)
+    end
+
+    def receive_data(data)
+      @recv_buffer.extract(data).each do |msg|
+        receive_task(msg)
+      end
+    end
+
+    def send_message(data)
+      send_data "#{data}#{TOKEN}"
+    end
+    
+    def receive_request
+      raise "receive_task not overridden by #{self.class.name}"
+    end
+  end
+end

data/lib/fpj/server.rb

@@ -2,6 +2,7 @@
 # go to eventmachine forum - see if anyone has already done this
 
 require 'fileutils'
+require 'timeout'
 
 module FileProcessingJob
   # FileProcessingJob::start_server creates an event machine server
@@ -70,21 +71,31 @@ module FileProcessingJob
     # create the directories if they do not exist
     Server.create_directories
 
-    # watch inbox directory
-    dw = DirectoryWatcher.new Server::config.inbox_directory, :glob => '*'
+    # manually load existing files in case there is a large backlog
+    # the directory_watcher interface does not handle large volumes
+    # well on startup
+    Dir.entries(Server::config.inbox_directory).each do |filename|
+      unless (File.directory?(File.join(Server::config.inbox_directory, filename)))
+        Server::Connection.push(filename) unless filename =~ /^\./
+      end
+    end
+
+    # watch the inbox directory for additions
+    dw = DirectoryWatcher.new Server::config.inbox_directory, :glob => '**.*', :pre_load => true
     dw.add_observer {|*args| 
       args.each {|event| 
         if (event.type == :added)
           filename = event.path.sub(Server::config.inbox_directory, '').sub(/^\//, '')
-          logger.debug "#{filename} discovered in inbox"
           Server::Connection.push(filename) 
         end
       }
     }
     dw.start
-
+    
     # run the server
     EM::run {
+
+      # start the server
       EM::start_server host, port, Server::Connection
       FileProcessingJob.logger.info "server started: #{host}:#{port}"
     }
@@ -124,10 +135,11 @@ module FileProcessingJob
       end
     end
 
-    class Connection < EM::Connection
+    class Connection < FileProcessingJob::Connection
       @@q = EM::Queue.new()
-      
+
       def self.push(message)
+        FileProcessingJob.logger.debug "#{message} discovered in inbox"
         @@q.push(message)
       end
 
@@ -138,39 +150,44 @@ module FileProcessingJob
       def logger
         FileProcessingJob.logger
       end
+
+      def send_file(filename)
+        logger.debug "Sending #{filename} to client"
+
+        send_message("filename: #{filename}")
+
+        FileUtils.move(File.join(self.config.inbox_directory, filename), File.join(self.config.processing_directory, filename))
+        streamer = EventMachine::FileStreamer.new(self, File.join(self.config.processing_directory, filename))
+        streamer.callback{
+          send_data TOKEN
+        }
+      end
       
       # inbound client connection
-      def receive_data data
+      def receive_task data
         data = data.strip
+        logger.debug("received command: #{data}")
         case (data)
-        when "next"
-          @@q.pop {|filename| send_data filename }
-        when /^get (.*)$/
-          filename = $1
-          streamer = EventMachine::FileStreamer.new(self, File.join(self.config.inbox_directory, filename))
-          streamer.callback{
-            FileUtils.move(File.join(self.config.inbox_directory, filename), File.join(self.config.processing_directory, filename))
-          }
-          # streamer.error {
-          #   logger.error "Unable to stream file #{filename} to client. The file can be found in the 'error' directory"
-          #   FileUtils.move(File.join(self.config.inbox_directory, filename), File.join(self.config.error_directory, filename))
-          # }
-        when /^processed (.*)$/
-          filename = $1
-          if (File.exists?(File.join(self.config.processed_directory, filename)))
-            logger.error "file #{filename} already exists in processed directory"
+          when "next"
+            @@q.pop {|filename| send_file(filename) }
+          when /^processed (.*)$/
+            filename = $1
+            logger.debug("finished processing #{filename}")
+            if (File.exists?(File.join(self.config.processed_directory, filename)))
+              logger.error "file #{filename} already exists in processed directory"
+            else
+              FileUtils.move(File.join(self.config.processing_directory, filename), File.join(self.config.processed_directory, filename))
+              logger.debug "finished processing #{filename}"
+            end
+            close_connection
+          when /^error (.*)$/
+            filename = $1
+            FileUtils.move(File.join(self.config.processing_directory, filename), File.join(self.config.error_directory, filename))
+            logger.error "Unable to stream file #{filename} to client. The file can be found in the 'error' directory"
+            close_connection
           else
-            FileUtils.move(File.join(self.config.processing_directory, filename), File.join(self.config.processed_directory, filename))
-            logger.debug "finished processing #{filename}"
+            puts "unknown command: '#{data}'"
           end
-          send_data "ack"
-        when /^error (.*)$/
-          filename = $1
-          FileUtils.move(File.join(self.config.processing_directory, filename), File.join(self.config.error_directory, filename))
-          logger.error "Unable to stream file #{filename} to client. The file can be found in the 'error' directory"
-        when "close"
-          close_connection
-        end
       end
   
       def bind

data/spec/spec_helper.rb

@@ -2,7 +2,6 @@ $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
 $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')))
 
 require 'rubygems'
-require 'aasm'
 require 'file-processing-job'
 require 'rspec'
 require 'rspec/autorun'

data/spec/unit/client_spec.rb

@@ -1,13 +1,13 @@
 require File.expand_path(File.join(File.dirname(__FILE__), '..', 'spec_helper'))
 
 class ErrorRaisingProcessor 
-  def receive_file data
+  def receive_file(filename, contents)
     raise "unknown error"
   end
 end
 
 class SuccessfulProcessor 
-  def receive_file data
+  def receive_file(filename, contents)
   end
 end
 
@@ -18,7 +18,6 @@ describe FileProcessingJob::Client do
       EventMachine.stub!(:send_data)
       @connection = FileProcessingJob::Client::Connection.new(11222)
       @filename = 'thefile.dat'
-      @connection.filename = @filename
     end
 
     describe "success" do
@@ -28,8 +27,8 @@ describe FileProcessingJob::Client do
       end
       
       it "should send back the success code" do
-        @connection.should_receive(:send_data).with("processed #{@filename}")
-        @connection.process_file
+        @connection.should_receive(:send_message).with("processed #{@filename}")
+        @connection.process(@filename, "ab")
       end
     end
     
@@ -41,12 +40,12 @@ describe FileProcessingJob::Client do
       
       it "should trap and log the exception" do
         FileProcessingJob.logger.should_receive(:error).twice # once for the human friendly message, once with the stack dump
-        lambda { @connection.process_file }.should_not raise_error
+        lambda { @connection.process(@filename, "ab") }.should_not raise_error
       end
 
       it "should send back an error message" do
-        @connection.should_receive(:send_data).with("error #{@filename}")
-        @connection.process_file
+        @connection.should_receive(:send_message).with("error #{@filename}")
+        @connection.process(@filename, "ab")
       end
     end
 

data/spec/unit/server_spec.rb

@@ -16,10 +16,13 @@ describe FileProcessingJob::Server do
       @qualified_filename = File.join(@config.inbox_directory, @filename)
     end
 
-    describe "get" do
-      it "should attempt to stream the file" do
-        EventMachine::FileStreamer.should_receive(:new).with(@connection, @qualified_filename).and_return(mock_streamer)
-        @connection.receive_data("get #{@filename}")
+    describe :send_file do
+      it "should attempt to send the file" do
+        File.should_receive(:read).and_return("ab")
+        @connection.should_receive(:send_message).with("filename: thefile.dat, size: 2, contents: ab")
+        FileUtils.should_receive(:move).with(File.join(@config.inbox_directory, @filename), File.join(@config.processing_directory, @filename))
+
+        @connection.send_file("thefile.dat")
       end
     end
 
@@ -27,7 +30,7 @@ describe FileProcessingJob::Server do
       it "should move the file to the error directory" do
         FileUtils.should_receive(:move).with(File.join(@config.processing_directory, @filename), File.join(@config.error_directory, @filename))
         @connection.logger.should_receive(:error)
-        @connection.receive_data("error #{@filename}")
+        @connection.receive_task("error #{@filename}")
       end
     end
   end
@@ -35,7 +38,6 @@ describe FileProcessingJob::Server do
   describe :create_directories do
     it "should create the directories if they do not exist" do
       Dir.should_receive(:exists?).exactly(4).times.with(an_instance_of(String)).and_return(false)
-      FileUtils.should_receive(:path).exactly(4).times.with(an_instance_of(String))
       FileProcessingJob::Server.create_directories
     end
     

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - cj2
@@ -17,24 +17,10 @@ cert_chain: []
 date: 2010-09-09 00:00:00 -06:00
 default_executable: 
 dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: aasm
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 1
-        version: "2.1"
-  type: :development
-  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: directory_watcher
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -44,11 +30,11 @@ dependencies:
         - 3
         version: "1.3"
   type: :development
-  version_requirements: *id002
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: eventmachine
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -58,7 +44,7 @@ dependencies:
         - 12
         version: "0.12"
   type: :development
-  version_requirements: *id003
+  version_requirements: *id002
 description: file-processing-job allows you to distribute the processing load of large files to clients across the network. It is a thin wrapper on top of the EventMachine library.
 email: cjgrimes@gmail.com
 executables: []
@@ -75,6 +61,7 @@ files:
 - examples/test_server.rb
 - lib/file-processing-job.rb
 - lib/fpj/client.rb
+- lib/fpj/connection.rb
 - lib/fpj/server.rb
 - spec/spec_helper.rb
 - spec/unit/client_spec.rb