jashmenn-basket 0.0.2

data/README.rdoc

@@ -0,0 +1,37 @@
+= Basket
+
+Easily process and sort a directory of files.
+
+= Example:
+
+  Basket.process("orders") do |file|
+    puts "we are processing #{file}"
+  end
+
+Assuming there are number of files in <tt>orders/inbox</tt> 
+* each file is +mv+'d to <tt>orders/pending</tt>
+* the block is called on each file
+* the file is +mv+'d to <tt>orders/archive</tt>
+
+See Basket#process for a list of all the options.
+
+= Install:
+
+  gem install jashmenn-basket --source http://gems.github.com
+
+= More examples:
+
+The output folder can be conditional based on the output of the block, as in
+the following example. In this case the default names of the folders are
++success+ and +fail+ based on the return value of the block being +true+ or
++false+.
+
+  :include:examples/02_conditional.rb 
+
+You can create arbitrary baskets for the output. If you specify <tt>:other</tt> then the files are *not* +mv+'d automatically. You must call the appropriate bang method on the file. For example:
+
+  :include:examples/03_other_baskets.rb  
+
+Baskets has (experimental) built-in support for doing parallel processing using <tt>forkoff</tt>. Example:
+
+  :include:examples/04_parallel.rb

data/Rakefile

@@ -0,0 +1,72 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+require 'spec/rake/spectask'
+require 'rake/rdoctask'
+require 'lib/ext/core'
+
+GEM = "basket"
+GEM_VERSION = "0.0.2"
+AUTHOR = "Nate Murray"
+EMAIL = "nate@natemurray.com"
+HOMEPAGE = "http://www.xcombinator.com"
+SUMMARY = "Easily process and sort a directory of files."
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = GEM_VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.rdoc"]
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  s.add_dependency('forkoff', '>= 0.0.1')
+  
+  s.require_path = 'lib'
+  s.autorequire = GEM
+  s.files = %w(README.rdoc Rakefile) + Dir.glob("{lib,spec,examples}/**/*").reject{|f| f =~ /(spec\/fixtures)/}
+end
+
+task :default => :spec
+
+desc "Run specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts = %w(-fs --color)
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "install the gem locally"
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
+end
+
+desc "create a gemspec file"
+task :make_spec do
+  File.open("#{GEM}.gemspec", "w") do |file|
+    file.puts spec.to_ruby
+  end
+end
+
+namespace :rdoc do
+  Rake::RDocTask.new :html do |rd|
+     rd.main = "README.rdoc"
+     rd.rdoc_dir = 'rdoc'
+     rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+  end
+  task :open do
+    system 'open ' + (:rdoc / 'index.html') if PLATFORM['darwin']
+  end
+end
+
+desc "run rstakeout"
+task :rstakeout do
+  exec "AUTOTEST=true rstakeout -t 1 -v \"spec spec --format=specdoc --color\" '*/**/*.rb'"
+end

data/examples/01_simple.rb

@@ -0,0 +1,8 @@
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'basket'
+def log(*args); p args; end
+
+# Default baskets: inbox, pending, archive
+Basket.process("orders") do |file|
+  log :processing, file
+end

data/examples/02_conditional.rb

@@ -0,0 +1,13 @@
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'basket'
+def log(*args); p args; end
+
+Basket.process("orders", :conditional => true, :logdev => "/dev/null") do |file, i|
+  if i % 2 == 0
+    log :success, file
+    true # returning true mv's file to +success+
+  else
+    log :fail, file
+    false # returning false mv's file to +faile+
+  end
+end

data/examples/03_other_baskets.rb

@@ -0,0 +1,17 @@
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'basket'
+def log(*args); p args; end
+
+Basket.process("orders", :inbox => "new", :pending => "work", :other => %w{good bad unknown}) do |file, i|
+  case i % 3 
+  when 0 
+    log :good, file
+    file.good! # mv's file to "orders/good"
+  when 1
+    log :bad, file
+    file.bad!
+  when 2
+    log :unknown, file
+    file.unknown!
+  end
+end

data/examples/04_parallel.rb

@@ -0,0 +1,10 @@
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'basket'
+def log(*args); p args; end
+
+
+# want to be able to process in parallel
+Basket.process("orders", :workers => 4) do |file|
+  log $$, :processing, file
+end
+

data/lib/basket.rb

@@ -0,0 +1,133 @@
+module Basket
+  VERSION = '0.0.1'
+
+  DIR = File.expand_path(File.dirname(File.expand_path(__FILE__)))
+  $:.unshift DIR
+
+  require 'fileutils'
+  require 'pp'
+  require 'ext/core'
+  require 'ext/metaid'
+  require 'has_logger'
+  require 'forkoff'
+
+  class << self
+    @logging = false
+    attr_accessor :logging
+  end
+  
+  # == Basket
+  # Process a directory of files, one at a time.
+  #
+  # == Overview
+  # Take +root+ folder and look for the directory +inbox+. Each file in
+  # +inbox+ is first +mv+'d to +pending+ and then yielded to +block+. Upon
+  # completion of the block the file is +mv+'d to a directory, as specifed in
+  # the options.
+  #
+  # == Options
+  # * <tt>:inbox</tt>: the name of the inbox folder (default +inbox+)
+  # * <tt>:pending</tt>: the name of the pending folder (default +pending+)
+  # * <tt>:archive</tt>: the name of the archive folder (default +archive+)
+  # * <tt>:other</tt>: if +other+ is specified then the files are *not* moved to the archive or any other directory automatically. You *must* specify where the file will go or it will remain in +inbox+. incompatable with +conditional+.
+  # * <tt>:logdev</tt>: device to log to. For example: <tt>STDOUT</tt> or <tt>"/path/to/log.log"</tt> (default: <tt>/dev/null</tt>)
+  # * <tt>:conditional</tt>: if +conditional+ is specified then then the result of the #process block is interpreted as boolean. if the result is +true+ then the file is mv'd to +success+ otherwise it is mv'd to +fail+
+  #
+  # == Block Arity
+  # If the block takes a single argument, then a string containing the path to
+  # the +pending+ file is yielded. If the block accepts two arugments then the
+  # file index is also yielded. Does not work for parallel processing with forkoff.
+  def self.process(input, opts={}, &block)
+    b = Base.new(input, opts)
+    b.process(&block)
+  end
+
+  class Base
+
+    INBOX   = "inbox"
+    PENDING = "pending"
+    ARCHIVE = "archive"
+
+    attr_accessor :root
+    attr_accessor :inbox
+    attr_accessor :pending
+    attr_accessor :other
+
+    include HasLogger
+    include FileUtils
+
+    def initialize(root, opts={})
+      @root = root
+      @inbox   = opts.delete(:inbox)   || INBOX
+      @pending = opts.delete(:pending) || PENDING
+      @archive = opts.delete(:archive) || ARCHIVE
+      @other   = opts.delete(:other)   || []
+      @logdev  = opts.delete(:logdev)  || "/dev/null"
+      @opts    = opts
+    end
+
+    def process(&block)
+      create_directories 
+      files = Dir[@root/@inbox/"*"]
+      logger.debug(["#{files.size} files in", @root/@inbox/"*"])
+      send_args = @opts[:workers] ? [:forkoff!, {:processes => @opts[:workers]}] : [:each]
+      i = 0
+      files.send(*send_args) do |file|
+        pending_file = @root/@pending/File.basename(file)
+        logger.info [:mv, file, pending_file]
+        mv file, pending_file
+
+        to_mix = mixin
+        pending_file.meta_eval { include to_mix }
+        result = block.arity > 1 ? block.call(pending_file, i) : block.call(pending_file)
+
+        if @opts[:conditional]
+          destination = result ? @root/"success" : @root/"fail"
+          logger.info [:mv, pending_file, destination]
+          mv pending_file, destination
+        elsif @other.size > 0
+          # don't mv anything
+        else
+          logger.info [:mv, pending_file, @root/@archive]
+          mv pending_file, @root/@archive
+        end
+        i += 1
+      end
+    end
+
+    protected
+    def create_directories
+      baskets.each do |dir|
+        logger.debug([:creating, @root/dir])
+        mkdir_p(@root/dir)
+      end
+    end
+
+    # create the Module that will be mixed in to the String representing the
+    # files this allows us to define exclamation methods that will move the
+    # files to a particular directory
+    def mixin # :nodoc:
+      our_baskets, our_root, our_logger = baskets, @root, logger # create local definitions for closures 
+      @mixin ||= begin 
+        movingMethods = Module.new
+        movingMethods.module_eval do
+          our_baskets.each do |basket|
+            define_method "#{basket}!" do
+              our_logger.info [:mv, self, our_root/basket]
+              FileUtils.mv self, our_root/basket
+            end
+          end
+        end
+        movingMethods
+      end
+    end
+
+    def baskets
+      baskets = [@inbox, @pending, @archive, @other]
+      baskets << ["success", "fail"] if @opts[:conditional]
+      baskets.flatten.compact
+    end
+
+  end
+end
+

data/lib/ext/core.rb

@@ -0,0 +1,11 @@
+class String
+  def /(o)
+    File.join(self, o.to_s)
+  end
+end
+
+class Symbol
+  def /(o)
+    File.join(self.to_s, o.to_s)
+  end
+end

data/lib/ext/metaid.rb

@@ -0,0 +1,15 @@
+ class Object
+   # The hidden singleton lurks behind everyone
+   def metaclass; class << self; self; end; end
+   def meta_eval &blk; metaclass.instance_eval &blk; end
+
+   # Adds methods to a metaclass
+   def meta_def name, &blk
+     meta_eval { define_method name, &blk }
+   end
+
+   # Defines an instance method within a class
+   def class_def name, &blk
+     class_eval { define_method name, &blk }
+   end
+ end

data/lib/has_logger.rb

@@ -0,0 +1,34 @@
+# Module for easy logging
+#
+# Example:
+#  
+#   class MyClass
+#      include HasLogger
+#
+#      # ...
+#      def do_something
+#        logger.info("I just did something!")
+#      end
+#   end
+require 'logger'
+
+module HasLogger
+  def self.included(mod)
+    mod.extend(ClassMethods)
+    mod.send :include, InstanceMethods
+  end
+
+  module ClassMethods
+  end
+
+  module InstanceMethods
+    def logger
+      @logger ||= begin 
+                logger = Logger.new(@logdev || STDOUT)
+                logger.formatter = Logger::Formatter.new
+                logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+                logger
+              end
+    end
+  end
+end

data/spec/basket_spec.rb

@@ -0,0 +1,122 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+def log *args
+  return false # comment to log these tests to stdout
+  p args
+end
+
+describe "Basket" do
+  include BasketFixtures
+
+  before(:each) do
+    create_fixture_files
+    @root = FIXTURES_DIR/"orders"
+  end
+
+  describe "Basket::Base" do 
+    describe "using default options" do
+      before(:each) do
+        @b = Basket::Base.new(FIXTURES_DIR/"orders", :logdev => "/dev/null")
+      end
+
+      it "should have root" do
+        @b.root.should eql(FIXTURES_DIR/"orders")
+      end
+
+      it "should create the needed directories" do
+        %w{pending archive}.each { |basket| File.exists?(@b.root/basket).should be_false }
+        @b.send(:create_directories)
+        %w{pending archive}.each { |basket| File.exists?(@b.root/basket).should be_true }
+      end
+    end
+
+    describe "making custom directories" do
+      before(:each) do
+        @all = %w{new work done foo bar baz}
+        @b = Basket::Base.new(FIXTURES_DIR/"orders", :inbox   => "new", 
+                                                     :pending => "work", 
+                                                     :archive => "done",
+                                                     :other   => @all,
+                                                     :logdev => "/dev/null")
+      end
+
+      it "should create them" do
+        @all.each { |basket| File.exists?(@b.root/basket).should be_false }
+        @b.send(:create_directories)
+        @all.each { |basket| File.exists?(@b.root/basket).should be_true }
+      end
+    end
+  end
+
+  describe "basic usage"  do
+    it "should process files" do
+      Dir["#{@root}/inbox/*"].size.should == 25 # check and make sure files are in the inbox
+
+      Basket.process(@root, :logdev => "/dev/null") do |file|
+        log :processing, file
+      end
+
+      Dir["#{@root}/inbox/*"  ].size.should == 0 
+      Dir["#{@root}/archive/*"].size.should == 25
+    end
+  end
+
+  describe "conditional usage" do
+    it "should process files" do
+      Dir["#{@root}/inbox/*"].size.should == 25 # check and make sure files are in the inbox
+
+      Basket.process(@root, :conditional => true, :logdev => "/dev/null") do |file, i|
+        if i % 2 == 0
+          log :success, file
+          true 
+        else
+          log :fail, file
+          false
+        end
+      end
+
+      Dir["#{@root}/inbox/*"   ].size.should == 0 
+      Dir["#{@root}/success/*" ].size.should == 13 
+      Dir["#{@root}/fail/*"    ].size.should == 12 
+    end
+  end
+
+  describe "custom baskets" do
+    it "should process files" do
+      create_fixture_files("new")
+      Dir["#{@root}/new/*"].size.should == 25 # check and make sure files are in the inbox
+
+      Basket.process(@root, :inbox => "new", :pending => "work", :other => %w{good bad unknown}) do |file, i|
+        case i % 3 
+        when 0 
+          log :good, file
+          file.good!
+        when 1
+          log :bad, file
+          file.bad!
+        when 2
+          log :unknown, file
+          file.unknown!
+        end
+      end
+
+      Dir["#{@root}/new/*"     ].size.should == 0
+      Dir["#{@root}/good/*"    ].size.should == 9
+      Dir["#{@root}/bad/*"     ].size.should == 8
+      Dir["#{@root}/unknown/*" ].size.should == 8
+    end
+  end
+
+  describe "parallel processing with forkoff" do
+    it "should process the files" do
+      create_fixture_files("inbox", 10)
+      Basket.process(@root, :workers => 2) do |file, i|
+        log $$, :processing, file, i
+      end
+    end
+  end
+
+  after(:each) do
+    cleanup_fixtures_dir
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+$TESTING=true
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+FIXTURES_DIR = File.join(File.dirname(__FILE__), "fixtures")
+require 'basket'
+
+module BasketFixtures
+  def fixtures_name
+    "order"
+  end
+
+  def fixtures_dir
+    FIXTURES_DIR/fixtures_name + "s"
+  end
+
+  def create_fixture_files inbox="inbox", howmany=25
+    cleanup_fixtures_dir
+    inbox_dir = fixtures_dir/inbox
+    FileUtils.mkdir_p(inbox_dir)
+    0.upto(howmany - 1) do |i|
+      filename = inbox_dir/"#{fixtures_name}_%05d" % i
+      FileUtils.touch(filename) unless File.exists?(filename)
+    end
+  end
+
+  def cleanup_fixtures_dir
+    FileUtils.rm_rf(fixtures_dir) if File.exists?(fixtures_dir)
+  end
+end
+

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: jashmenn-basket
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+platform: ruby
+authors: 
+- Nate Murray
+autorequire: basket
+bindir: bin
+cert_chain: []
+
+date: 2009-04-10 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: forkoff
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.0.1
+    version: 
+description: Easily process and sort a directory of files.
+email: nate@natemurray.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- lib/basket.rb
+- lib/ext
+- lib/ext/core.rb
+- lib/ext/metaid.rb
+- lib/has_logger.rb
+- spec/basket_spec.rb
+- spec/spec_helper.rb
+- examples/01_simple.rb
+- examples/02_conditional.rb
+- examples/03_other_baskets.rb
+- examples/04_parallel.rb
+has_rdoc: true
+homepage: http://www.xcombinator.com
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Easily process and sort a directory of files.
+test_files: []
+