shovel 0.0.1

data/History.txt

@@ -0,0 +1,4 @@
+== 0.0.1 2009-03-02
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,8 @@
+History.txt
+Manifest.txt
+README.rdoc
+lib/dir_ext.rb
+lib/flow.rb
+lib/log.rb
+lib/shovel.rb
+spec/example.rb

data/README.rdoc

@@ -0,0 +1,52 @@
+= shovel
+
+* Shovel http://bitbucket.org/mediashelf/shovel
+
+== DESCRIPTION:
+
+A domain specific language for configuring ingestor processes
+
+== FEATURES/PROBLEMS:
+
+This module takes care of the drudgery of directory scanning, archive extraction and tempfile management, allowing the developer to implement the fiddly bits without thinking about the tedious bits.
+
+More archive formats could be pretty easily supported. Some refactoring of the archive extraction code is probably a good idea too.
+
+== SYNOPSIS:
+
+  include Shovel
+
+== REQUIREMENTS:
+
+* directory_watcher
+* rubyzip
+* archive-tar-minitar
+
+== INSTALL:
+
+* sudo gem install
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Mediashelf LLC
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/dir_ext.rb

@@ -0,0 +1,19 @@
+require 'tmpdir'
+class Dir
+  #a handy method to create a tmpdir, which cleans up after itself.
+  # Dir.mktmp do |dir|
+  #   puts "i'm doing neat stuff"
+  # end
+  #
+  #Even if your block throws an exception, the temp directory will be deleted.
+  def self.mktmp(prefix="tmpdir", &block)
+    dirname = File.join(Dir.tmpdir, "%s.%s"%[prefix, Time.now.strftime("%s")])
+    FileUtils.mkdir_p dirname
+    begin
+      yield dirname
+    ensure
+      FileUtils.rm_rf dirname
+    end
+    dirname
+  end
+end

data/lib/flow.rb

@@ -0,0 +1,62 @@
+require 'log'
+require 'forwardable'
+module MediaShelf
+  module Shovel
+    #This class represents a "workflow" for a file contained in an ingest file being
+    #processed by Shovel
+    #See +Shovel+ for more information and an example
+    class Flow
+      extend Forwardable
+      def_delegators Log.instance, :d, :e, :w, :i
+      attr_reader :reaction_map, :exception_map
+
+
+      def initialize #:nodoc:
+        @reaction_map={:stable=>[], :added=>[], :removed=>[], :modified=>[]}
+        @exception_map = {}
+      end
+      #add a process to the flow, to be executed by the Shovel when the
+      #supplied event occurrs.
+      #
+      #event:: the event type, one of :stable, :added, :removed or :modified
+      #block:: your process. the process will be passed hte file extracted \
+      #from your archive (most likely).       
+      #
+      #This method (and on_exception) returns self, so calls 
+      #to add_reaction can be chained.
+      def add_reaction(event, &block)
+        @reaction_map[event] <<  block
+        return self
+      end
+      #add a process to the flow, to be executed by the Shovel when the
+      #supplied Exception occurrs.
+      #klazz:: a class which might be raised at some point in the ingest process
+      #block:: what you want to do about it. The block is passed the file,  \
+      #and the exception instance.
+      #
+      #This method (and add_reaction) returns self, so calls 
+      #to add_reaction can be chained.
+      def on_exception(klazz, &block)
+        @exception_map[klazz.name.to_sym]=block
+        return self
+      end
+
+      #The exception handler.  It will try to find processes mapped to
+      #the given exception, and invoke on them. If it can't find a handler
+      #it just logs the exception and bails.
+      def handle_exception(f,x)
+        if @exception_map[x.class.name.to_sym]
+          p = @exception_map[x.class.name.to_sym]
+          p.call(f, x)
+        else
+          log_exception(f,x)
+        end
+      end
+
+      def log_exception(f,x)#:nodoc:
+        e("Error : #{f} #{x.message}. Stack trace follows:")
+        e(x.backtrace.join("\n"))
+      end
+    end
+  end
+end

data/lib/log.rb

@@ -0,0 +1,34 @@
+require 'singleton'
+require 'logger'
+module MediaShelf
+  # A singleton logger. That way we can use the same instance all over the place.
+  # Singleton claims to be threadsafe. 
+  class Log 
+    include Singleton
+    def silence!
+      @logger = Logger.new(File.open('/dev/null', 'w')) 
+    end
+    def unsilence!
+      @logger = Logger.new(STDOUT)
+    end
+    def initialize
+      @logger = Logger.new(STDOUT)
+      @logger.datetime_format = "%y%m%d %H:%M:%S"
+    end
+    def d(msg)
+      @logger.debug msg if @logger.level == Logger::DEBUG
+    end
+    def e(msg)
+      @logger.error msg
+    end
+    def w(msg)
+      @logger.warn msg
+    end
+    def i(msg)
+      @logger.info msg
+    end
+    def f(msg)
+      @logger.fatal msg
+    end
+  end
+end

data/lib/shovel.rb

@@ -0,0 +1,137 @@
+$:.unshift(File.dirname(__FILE__)) unless
+$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+require 'rubygems'
+gem 'directory_watcher'
+gem 'rubyzip'
+gem 'archive-tar-minitar'
+require 'zip/zip'
+require 'directory_watcher'
+require 'log'
+require 'flow'
+require 'dir_ext'
+require 'zlib'
+require 'archive/tar/minitar'
+
+module Shovel
+  VERSION = '0.0.1'
+end
+
+module MediaShelf
+
+  # = Shovel
+  # This module can be mixed into a class that would like to react to 
+  # file events in a particular directory.   For a complete
+  # example see spec/example.rb
+  # :include:../spec/example.rb
+  module Shovel
+    include Archive::Tar::Minitar
+    
+    def d(s) #:nodoc:
+      Log.instance.d(s) 
+    end
+
+    
+    def update(*events)#:nodoc:
+      d("events received: #{events.inspect}")
+      events.each do |e|
+        d "found : "+@flow.reaction_map[e.type].inspect
+        @flow.reaction_map[e.type].each {|x| 
+          begin
+            d("executing: #{x}")
+            x.call(e.path)
+          rescue Exception=>x
+            @flow.handle_exception(e.path, x)
+          end
+        }
+      end
+    end
+
+    #This method is used to configure the directory scanner. A +Flow+ object is returned,
+    #so constructs like
+    #
+    # configure_process.add_reaction.add_reaction
+    #
+    #can be created. This method will throw an exception if it's invoked 
+    #more than once on the same instance.
+    #
+    #dir:: directory to scan, will be created if it doesn't exist
+    #interval:: how often to scan
+    #stable_cycles:: how many intervals should the file remain unchanged to fire a stable event
+    #glob:: array of globs to scan for
+    #pre_load:: whether to suppress file events for files that already exist when the scanner is started. Defaults to true
+    def configure_process(dir, interval, stable_cycles, glob, pre_load=true )
+      raise ArgumentError.new("cannot pass a regex in here, globs only please") if glob.is_a?(Regexp)
+      raise ConfigurationError unless @dw.nil?
+      Log.instance.d("configured to scan #{dir} every #{interval}s for #{glob.inspect}. pre loading existing files? #{pre_load}")
+      @dw = DirectoryWatcher.new(dir, :pre_load=>pre_load, :glob=>glob, :stable=>stable_cycles, :interval=>interval) 
+      @flow = Flow.new
+      @dw.add_observer(self)
+      @flow
+
+    end
+
+    #Starts the directory scanner process.
+    def start
+      @dw.start
+      @dw.join
+    end
+
+    #this method knows how to extract files from a standard .zip file. The files 
+    #themselves are extracted into an opaque tmp directory, which is cleaned up 
+    #after the block is yielded.
+    #
+    #filename:: the name of the zip archive
+    #process:: A block which will be passed each entry in the zipfile for further pocessing
+    #
+    #Example:
+    # unzip('foo.zip'){|file| puts "i'm processing #{file}"}
+    def unzip(filename, &process)
+      d("unzipping #{filename}")
+      Dir.mktmp do |d|
+        Zip::ZipFile.foreach(filename) do |entry|
+          d "processing #{entry}"
+          FileUtils.mkdir_p(File.dirname(File.join(d, entry.name)))
+          entry.extract(File.join(d, entry.name)){|x| w "warn: #{x} exists!"}
+          yield File.join(d, entry.name)
+        end
+      end
+    end
+    #see +unzip+. Same thing, except for tar archives
+    def untar(filename, &process)
+      d("untarring #{filename}")
+      untar_gz(filename, false,  &process )
+    end
+    #see +unzip+. Same thing, except for gzipped tar archives.
+    def untar_gz(filename,  gzipped=true, &process)
+      d("targunzipping #{filename}")
+      tgz = gzipped ?  Zlib::GzipReader.new(File.open(filename, 'rb')) : filename
+      Dir.mktmp do |d|
+        Archive::Tar::Minitar.unpack(tgz,d) do |event, name, stats|
+          yield File.join(d,name)  if event == :file_done
+        end
+      end
+    end
+    #see +unzip+. This method will try to figure out what kind of archive you passed it,
+    #then call the appropriate method.
+    def extract(filename, &process)
+      case filename
+      when /(.*\.tar.gz|.*\.tgz)/
+        untar_gz(filename, &process)
+      when /.*\.zip/
+        unzip( filename, &process)
+      when /.*\.tar/
+        untar(filename, &process)
+      else
+        raise "Couldn't figure out what kind of archive #{filename} was..."
+      end
+    end
+
+    #Raised if configure_process is called more than once.
+    class ConfigurationError < Exception
+      def initialize
+        super("The shovel can only be configured once.")
+      end
+    end
+  end
+
+end

data/spec/example.rb

@@ -0,0 +1,53 @@
+require 'rubygems'
+gem 'shovel'
+require 'ruby-debug'
+require 'shovel'
+
+class Example
+  #include the shovel mixin
+  include MediaShelf::Shovel
+
+  def initialize
+    super
+    #make the ingest directory.
+    root = File.dirname(__FILE__)
+    indir = File.join(root, 'ingest')
+    faildir= File.join(root, 'failboat')
+    FileUtils.mkdir_p faildir unless File.exist?(faildir)
+
+    #configure an ingest process to scan our ingest directory for compressed archives every 10s, consider an archive
+    #stable if it doesn't change in 10s, and don't suppress file events for files that are already in there(!)
+    flow = configure_process(indir, 4, 1, %w(**/*.zip **/*.tgz **/*.tar.gz **/*.tar), false)
+    #let's react to stable events by...
+    flow.add_reaction(:stable) do |file|
+      #extracting the file...
+      extract(file) do |inner|
+        #and doing some stuff to it
+        puts "i'm doing stuff to #{inner} of size #{File.size(inner)}"
+        #and possibly raising an exception, so we can demonstrate exception handling
+        raise "This file is Exceptional! !" if file =~ /.*\.gz/
+      end
+    end
+    flow.add_reaction(:stable) do |file|
+      #now let's delete it. these reactions are executed _in_order!
+      FileUtils.rm(file, :verbose=>true)
+    end
+    flow.add_reaction(:removed) do |file|
+      d("so long #{file}!")
+    end
+
+
+    flow.on_exception(RuntimeError) do |f, e|
+      d "URF!, an exception was thrown on #{f}! Welcome to the FAILboat!"
+      d e.message
+      FileUtils.mv(f, File.join(faildir, "%s.%d"%[File.basename(f),Time.now]), :verbose=>true) 
+    end
+  end
+
+
+end
+
+#create our example class
+ex = Example.new
+#and start it.
+ex.start

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification 
+name: shovel
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- McClain Looney
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-04 00:00:00 -06:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: directory_watcher
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rubyzip
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.9.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: archive-tar-minitar
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.5.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: newgem
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.3
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.9.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.9.1
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.0
+    version: 
+description: A domain specific language for configuring ingestor processes
+email: 
+- mcclain.looney@yourmediashelf.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.rdoc
+files: 
+- History.txt
+- Manifest.txt
+- README.rdoc
+- lib/dir_ext.rb
+- lib/flow.rb
+- lib/log.rb
+- lib/shovel.rb
+- spec/example.rb
+has_rdoc: true
+homepage: Shovel http://bitbucket.org/mediashelf/shovel
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: rubyfedora
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: A domain specific language for configuring ingestor processes
+test_files: []
+