fileq 0.1.3

data/lib/job.rb

@@ -0,0 +1,181 @@
+# FileQ 
+require 'yaml'
+#require 'fcntl'
+
+module Xxeo
+
+  class WrongStatus < StandardError; end
+
+  class Job
+
+    def Job.create(queue, name, path, status)
+      raise unless queue.class == Xxeo::FileQ
+
+      Job.new(queue, name, path, status)
+    end
+
+    def name
+      return @name
+    end
+
+    def data
+      raise WrongStatus.new('cannot disown a non-owned job') unless @own
+      data = nil
+      File.open(@path + '/data') { |f| data = f.read }
+      return data
+    end
+
+    def owning_pid
+      @owning_pid 
+    end
+
+    def is_file?
+      meta = read_meta
+      raise unless meta
+      return meta[:XX_type] == 'file'
+    end
+
+    def original_pathname
+      meta = read_meta
+      raise unless meta
+      return meta[:XX_original_file_name] if meta[:XX_type] == 'file'
+      return nil
+    end
+
+    def read_meta
+      return @fq.meta_for_job(@name)
+    end
+
+    def own?
+      @own
+    end
+
+    def disown
+      raise WrongStatus.new('cannot disown a non-owned job') unless @own
+      @own = false
+      File.unlink(@path + '/pid')
+      @owning_pid = nil
+    end
+
+    def set_status(path, status)
+      @path = path
+      @status = status
+    end
+
+    def pull
+      return self if @own 
+
+      # This is kinda wonky, obviously
+      # since the @fq already called set_as_active on an object
+      if @fq.pull_job(@name)
+        @own = true
+        @path = @fq.run_que_path + @name
+        @status = ST_RUN
+        @owning_pid = $$
+        return self
+      end
+      return nil
+    end
+
+    def set_as_active
+      @own = true
+      # Record
+      @path = @fq.run_que_path + @name
+      @status = ST_RUN
+      File.open(@path + '/pid', "w") { |f| f.write("#{$$}\n") }
+      @owning_pid = $$
+    end
+
+    # TODO: this could be stale
+    def status
+      return @status if @own
+      return @fq.status(@name)
+    end
+
+    # TODO: this could be stale
+    def status_mesg
+      return @status_mesg if @own
+      return @fq.status_mesg_for_job(@name)
+    end
+
+    # The @fq locks and then does a callback here
+    def callback_status_mesg
+      data = ''
+      File.open(@path + '/status', "r") do
+        |f|
+        data = f.read
+      end
+      return data
+    end
+
+    def status_mesg=(msg)
+      raise WrongStatus.new('cannot set status mesg on non-owned job') unless @own
+      @status_mesg = msg
+      File.open(@path + '/status', "w") { |f| f.write(msg) }
+    end
+
+    def status_all
+      return [@status, @owning_pid, status_mesg]
+    end
+
+    def log(msg)
+      raise WrongStatus.new('cannot log on non-owned job') unless @own
+      File.open(@path + '/log', "a") do
+        |f|
+        log_msg = Time.now.to_s + " == " + msg + "\n"
+        f.write(log_msg)
+        @logs << log_msg
+      end
+    end
+
+    def read_log
+      return @logs.join('') if @own
+      return @fq.status_logs_for_job(@name)
+    end
+
+    # The @fq locks and then does a callback here
+    def callback_read_logs
+      data = ''
+      File.open(@path + '/log', "r") do
+        |f|
+        data = f.read
+      end
+      return data
+    end
+
+    def mark_done
+      raise WrongStatus.new('cannot mark job as done on non-owned job') unless @own
+      status_mesg = "finishing"
+      @fq.mark_job_done(self)
+    end
+
+    def mark_error
+      raise WrongStatus.new('cannot mark job as error on non-owned job') unless @own
+      status_mesg = "finishing"
+      @fq.mark_job_error(self)
+    end
+
+    def reinsert
+      raise WrongStatus.new('cannot reinsert on non-owned job') unless @own
+      status_mesg = "reinserting"
+      log("reinserting job")
+      @fq.reinsert_job(self)
+    end
+
+    private
+
+    def initialize(queue, name, path, status)
+      @fq = queue
+      @name = name
+      @path = path
+      @own = false
+      @owning_pid = nil
+      @status = status
+      @status_mesg = ''
+      @logs = []
+    end
+
+  end
+
+end
+

data/lib/lockfile.rb

@@ -0,0 +1,88 @@
+# LockFile` 
+#
+# This assumes that a single threaded process is using
+# this object to use a file on disk as a lock/mutex
+# for synchronization.
+#
+#   class User < ActiveRecord::Base
+#     acts_as_audited
+#   end
+#
+# See <tt>CollectiveIdea::Acts::Audited::ClassMethods#acts_as_audited</tt>
+# for configuration options
+#http://rails.techno-weenie.net/tip/2005/11/19/validate_your_forms_with_a_table_less_model
+#http://lists.vanruby.com/pipermail/discuss/2006-January/000050.html
+
+# class Order < InActiveRecord
+# column :id,            :integer
+# column :name,       :string
+# column :address,    :string
+#  ...
+#  validates_presence_of :name, :address ....
+#
+#
+
+require 'fcntl'
+
+module Xxeo
+
+  class LockFile
+
+    def initialize(pathname, options = {})
+      options[:env]      ||= 'development'
+      
+      raise "lockfile not writable" if not File.writable? pathname
+
+      @f = File.open(pathname, 'r')
+
+      raise "could not open file" if not @f
+
+      @lock_count = 0
+    end
+
+    # We allow a process that holds a lock to recursively
+    # acquire the lock, the allows the higher level programs
+    # to not have to keep track of the lock status
+    # This is becase the flock mechanism doesn't do reference
+    # counting.
+    
+    def lock
+      # already locked, just increase the ref count
+      if @lock_count > 0
+        @lock_count += 1
+        return
+      end
+
+      # THIS COULD BLOCK
+      @f.flock(File::LOCK_EX)
+      @lock_count = 1
+    end
+
+    def unlock
+      if @lock_count > 1
+        @lock_count -= 1
+        return
+      end
+
+      if @lock_count == 1
+        @f.flock(File::LOCK_UN)
+        @lock_count -= 1
+        return
+      end
+
+      if @lock_count < 1
+        raise "Attempt to unlock an unlocked LockFile"
+      end
+    end
+
+    def locked?
+      return @lock_count > 0
+    end
+
+    def lock_count
+      return @lock_count
+    end
+
+  end
+
+end

data/test/fileq_createdir_tests.rb

@@ -0,0 +1,83 @@
+#require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper'))
+
+
+class FileQCreateDirTests < Test::Unit::TestCase
+
+  def setup
+    @fq = Xxeo::FileQ.new('test', {:dir => 'test/test_fq'})
+    @fq.create_queue_dirs
+  end
+
+  def teardown
+    FileUtils.rm_rf 'test/test_fq' 
+  end
+
+  def test_create
+    assert(FileTest.directory?('test/test_fq')) 
+    assert(FileTest.writable?('test/test_fq')) 
+    assert(FileTest.readable?('test/test_fq/_lock')) 
+    assert(FileTest.writable?('test/test_fq/_log')) 
+    assert(FileTest.directory?('test/test_fq/_tmp')) 
+    assert(FileTest.directory?('test/test_fq/que')) 
+    assert(FileTest.directory?('test/test_fq/run')) 
+    assert(FileTest.directory?('test/test_fq/pause')) 
+    assert(FileTest.directory?('test/test_fq/done')) 
+    assert(FileTest.directory?('test/test_fq/_err')) 
+  end
+
+  def test_create_verify
+    assert(@fq.verify_store, 'Failed to verify')
+    assert_equal('', @fq.last_error, 'Failed to have empty error message')
+  end
+
+  def test_bad_lock_file
+    FileUtils.rm_rf('test/test_fq/_lock')
+    assert(! @fq.verify_store, 'Failed to verify as false')
+    assert_equal('bad queue dir: file \'_lock\' does not exist', @fq.last_error, 'Failed to have correct error message')
+  end
+
+  #TODO: test lock file writable
+
+  def test_bad_log_file
+    FileUtils.rm_rf('test/test_fq/_log')
+    assert(! @fq.verify_store, 'Failed to verify as false')
+    assert_equal('bad queue dir: file \'_log\' does not exist', @fq.last_error, 'Failed to have correct error message')
+  end
+
+  def test_bad_tmp_dir
+    FileUtils.rm_rf('test/test_fq/_tmp')
+    assert(! @fq.verify_store, 'Failed to verify as false')
+    assert_equal('bad queue dir: \'_tmp\' not a directory', @fq.last_error, 'Failed to have correct error message')
+  end
+
+  def test_bad_q_dir
+    FileUtils.rm_rf('test/test_fq/que')
+    assert(! @fq.verify_store, 'Failed to verify as false')
+    assert_equal('bad queue dir: \'que\' not a directory', @fq.last_error, 'Failed to have correct error message')
+  end
+
+  def test_bad_run_dir
+    FileUtils.rm_rf('test/test_fq/run')
+    assert(! @fq.verify_store, 'Failed to verify as false')
+    assert_equal('bad queue dir: \'run\' not a directory', @fq.last_error, 'Failed to have correct error message')
+  end
+
+  def test_bad_pause_dir
+    FileUtils.rm_rf('test/test_fq/pause')
+    assert(! @fq.verify_store, 'Failed to verify as false')
+    assert_equal('bad queue dir: \'pause\' not a directory', @fq.last_error, 'Failed to have correct error message')
+  end
+
+  def test_bad_done_dir
+    FileUtils.rm_rf('test/test_fq/done')
+    assert(! @fq.verify_store, 'Failed to verify as false')
+    assert_equal('bad queue dir: \'done\' not a directory', @fq.last_error, 'Failed to have correct error message')
+  end
+
+  def test_bad_err_dir
+    FileUtils.rm_rf('test/test_fq/_err')
+    assert(! @fq.verify_store, 'Failed to verify as false')
+    assert_equal('bad queue dir: \'_err\' not a directory', @fq.last_error, 'Failed to have correct error message')
+  end
+
+end

data/test/fileq_find_tests.rb

@@ -0,0 +1,74 @@
+
+class FileQFindTests < Test::Unit::TestCase
+
+  def setup
+    @fq = Xxeo::FileQ.new('test', {:dir => 'test/test_fq'})
+    @fq.create_queue_dirs
+
+    @tmp_fname1 = 'tmp_file1'
+    @tmp_name1 = 'test/test_fq/' + @tmp_fname1
+    @data1 = "blah blah1\nblah blah1\n"
+    File.open(@tmp_name1, "w") { |f| f.write(@data1) }
+
+    @tmp_fname2 = 'tmp_file2'
+    @tmp_name2 = 'test/test_fq/' + @tmp_fname2
+    @data2 = "blah blah2\nblah blah2\n"
+    File.open(@tmp_name2, "w") { |f| f.write(@data2) }
+
+    @job1 = @fq.insert_data('This is a test')
+  end
+
+  def teardown
+    FileUtils.rm_rf 'test/test_fq' 
+  end
+
+  def test_find
+    assert(@fq.find_job(@job1.name))
+  end
+
+  def test_find_fail
+    assert(!@fq.find_job('dldldld'))
+  end
+
+  def test_find_que
+    job = @fq.find_job(@job1.name)
+    assert(job)
+    assert_equal(job.status, Xxeo::ST_QUEUED)
+    assert_equal(false, job.own?)
+  end
+
+  def test_find_run
+    @job1.pull
+    job = @fq.find_job(@job1.name)
+    assert(job)
+    assert_equal(job.status, Xxeo::ST_RUN)
+    assert_equal(false, job.own?)
+  end
+
+  def test_find_and_run
+    job = @fq.find_job(@job1.name)
+    assert(job)
+    assert(job.pull, @fq.read_log)
+    assert_equal(job.status, Xxeo::ST_RUN)
+    assert_equal(true, job.own?)
+  end
+
+  def test_find_done
+    @job1.pull
+    @job1.mark_done
+    job = @fq.find_job(@job1.name)
+    assert(job)
+    assert_equal(job.status, Xxeo::ST_DONE)
+    assert_equal(false, job.own?)
+  end
+
+  def test_find_err
+    @job1.pull
+    @job1.mark_error
+    job = @fq.find_job(@job1.name)
+    assert(job)
+    assert_equal(job.status, Xxeo::ST_ERROR)
+    assert_equal(false, job.own?)
+  end
+
+end

data/test/fileq_first_tests.rb

@@ -0,0 +1,33 @@
+#require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper'))
+
+class FileQFirstTest < Test::Unit::TestCase
+
+  def setup
+    FileUtils.mkdir 'test/test_fq' 
+    FileUtils.touch 'test/test_fq/_lock' 
+    FileUtils.touch 'test/test_fq/_log' 
+    FileUtils.mkdir 'test/test_fq/_tmp' 
+    FileUtils.mkdir 'test/test_fq/que' 
+    FileUtils.mkdir 'test/test_fq/run' 
+    FileUtils.mkdir 'test/test_fq/pause' 
+    FileUtils.mkdir 'test/test_fq/done' 
+    FileUtils.mkdir 'test/test_fq/_err' 
+    File.open('test/test_fq/lock', "w") { |f| f.close }
+  end
+
+  def teardown
+    FileUtils.rm_rf 'test/test_fq' 
+  end
+
+  def test_instantiate
+    fq = Xxeo::FileQ.new('test', { :dir => 'test/test_fq'})
+    assert_not_nil(fq)
+  end
+
+  def test_instantiate_verify
+    fq = Xxeo::FileQ.new('test', { :dir => 'test/test_fq'})
+    assert_not_nil(fq)
+    assert(fq.verify_store)
+  end
+
+end