imw 0.1.1 → 0.2.0

data/lib/imw/utils/log.rb

@@ -1,16 +1,16 @@
 require 'logger'
 
 module IMW
+
+  # Default log file.
   LOG_FILE_DESTINATION = STDERR             unless defined?(LOG_FILE_DESTINATION)
+  
   LOG_TIMEFORMAT       = "%Y%m%d-%H:%M:%S " unless defined?(LOG_TIMEFORMAT)
 
   class << self; attr_accessor :log end
-  #
-  # Create a Logger and point it at LOG_FILE_DESTINATION
-  #
-  # LOG_FILE_DESTINATION is STDOUT by default; redefine it in your
-  # ~/.imwrc, or set IMW.log yourself, if that's not cool.
-  #
+  
+  # Create a Logger and point it at IMW::LOG_FILE_DESTINATION which is
+  # set in ~/.imwrc and defaults to STDERR.
   def self.instantiate_logger!
     IMW.log ||= Logger.new(LOG_FILE_DESTINATION)
     IMW.log.datetime_format = "%Y%m%d-%H:%M:%S "
@@ -18,15 +18,19 @@ module IMW
   end
 
   def announce *events
-    options = events.extract_options!
+    options = events.flatten.extract_options!
     options.reverse_merge! :level => Logger::INFO
-    # puts [options, events ].inspect, "*"*76
     IMW.log.add options[:level], events.join("\n")
   end
   def banner *events
-    options = events.extract_options!
+    options = events.flatten.extract_options!
     options.reverse_merge! :level => Logger::INFO
-    ["*"*75, events, "*"*75].flatten.each{|ev| announce(ev, options) }
+    announce(["*"*75, events, "*"*75], options)
+  end
+  def warn *events
+    options = events.flatten.extract_options!
+    options.reverse_merge! :level => Logger::WARN
+    announce events, options
   end
 
   PROGRESS_TRACKERS = {}
@@ -61,7 +65,4 @@ module IMW
   end
 end
 
-#
-# Make the default logger
-#
 IMW.instantiate_logger!

data/lib/imw/utils/misc.rb

@@ -1,10 +1,4 @@
 module IMW
-  # Return a string representing the current UTC time in the IMW
-  # format.
-  def self.current_utc_time_string
-    Time.now.utc.strftime(IMW::STRFTIME_FORMAT)
-  end
-
 
   # A simple counter.  The +value+ and +add+ methods read and
   # increment the counter's value.

data/lib/imw/utils/paths.rb

@@ -1,3 +1,5 @@
+require 'pathname'
+
 module IMW
 
   # Implements methods designed to work with an object's
@@ -9,19 +11,48 @@ module IMW
   # <tt>@paths</tt>.
   module Paths
 
-    # Expands a shorthand workflow path specification to an
-    # actual file path.
+    # Expands a shorthand workflow path specification to an actual
+    # file path.  Strings are interpreted literally but symbols are
+    # first resolved to the paths they represent.
+    #
+    #   add_path :foo, '~/whoa'
+    #   path_to :foo, 'my_thing'
+    #   => '~/whoa/my_thing'
     #
-    #   add_path :mlb_08, 'gd2.mlb.com/components/game/mlb/year_2008'
-    #   path_to :ripd, :mlb_08, 'month_06', 'day_08', 'miniscoreboard.xml'
-    #   => (...)/data/ripd/gd2.mlb.com/components/game/mlb/year_2008/month_06/day_08/miniscoreboard.xml
+    # @param [String, Symbol] pathsegs the path segments to join
+    # @return [String] the resulting expanded path
     def path_to *pathsegs
-      begin
-        path = Pathname.new path_to_helper(*pathsegs)
-        path.absolute? ? File.expand_path(path) : path.to_s
-      rescue Exception => e
-        raise("Can't find path to '#{pathsegs}': #{e}");
-      end
+      path = Pathname.new path_to_helper(*pathsegs)
+      path.absolute? ? File.expand_path(path) : path.to_s
+    end
+
+    # Return the presently defined paths for this object.
+    #
+    # @return [Hash]
+    def paths
+      @paths ||= {}
+    end
+
+    # Adds a symbolic path for expansion by +path_to+.
+    # 
+    #   add_path :foo, '~/whoa'
+    #   add_path :bar, :foo,   'baz'
+    #   path_to :bar
+    #   => '~/whoa/baz'
+    #
+    # @param [Symbol] sym the name of the path to store
+    # @param [Symbol, String] pathsegs the path segments to use to define the path to the name
+    # @return [String] the resulting path
+    def add_path sym, *pathsegs
+      paths[sym] = pathsegs.flatten
+      path_to(sym)
+    end
+
+    # Removes a symbolic path for expansion by +path_to+.
+    #
+    # @param [Symbol] sym the stored path symbol to remove
+    def remove_path sym
+      paths.delete sym if paths.include? sym
     end
 
     private
@@ -29,7 +60,7 @@ module IMW
       # +path_to_helper+ handles the recursive calls for +path_to+.
       expanded = pathsegs.flatten.compact.map do |pathseg|
         case
-        when pathseg.is_a?(Symbol) && @paths.include?(pathseg)     then path_to(@paths[pathseg])
+        when pathseg.is_a?(Symbol) && paths.include?(pathseg)      then path_to(paths[pathseg])
         when pathseg.is_a?(Symbol) && IMW::PATHS.include?(pathseg) then path_to(IMW::PATHS[pathseg])          
         when pathseg.is_a?(Symbol)                                 then raise IMW::PathError.new("No path expansion set for #{pathseg.inspect}")
         else pathseg
@@ -37,29 +68,70 @@ module IMW
       end
       File.join(*expanded)
     end
-    public
+  end
 
-    # Adds a symbolic path for expansion by +path_to+.
-    def add_path sym, *pathsegs
-      @paths[sym] = pathsegs.flatten
-    end
 
-    # Removes a symbolic path for expansion by +path_to+.
-    def remove_path sym
-      @paths.delete sym if @paths.include? sym
-    end
-  end
+  # Default paths for the IMW.  Chosen to make sense on most *NIX
+  # distributions.
+  DEFAULT_PATHS = {
+    :home         => ENV['HOME'],
+    :data_root    => "/var/lib/imw",
+    :log_root     => "/var/log/imw",
+    :scripts_root => "/usr/share/imw",
+    :tmp_root     => "/tmp/imw",
+
+    # the imw library
+    :imw_root  => File.expand_path(File.dirname(__FILE__) + "/../../.."),
+    :imw_bin   => [:imw_root, 'bin'],
+    :imw_etc   => [:imw_root, 'etc'],
+    :imw_lib   => [:imw_root, 'lib'],
+
+    # workflow
+    :ripd_root  => [:data_root, 'ripd'],
+    :rawd_root  => [:data_root, 'rawd'],
+    :fixd_root  => [:data_root, 'fixd'],
+    :pkgd_root  => [:data_root, 'pkgd']
+  }
+  defined?(PATHS) ? PATHS.reverse_merge!(DEFAULT_PATHS) : PATHS = DEFAULT_PATHS
 
+  # Expands a shorthand workflow path specification to an actual
+  # file path.  Strings are interpreted literally but symbols are
+  # first resolved to the paths they represent.
+  #
+  #   IMW.add_path :foo, '~/whoa'
+  #   IMW.path_to :foo, 'my_thing'
+  #   => '~/whoa/my_thing'
+  #
+  # @param [String, Symbol] pathsegs the path segments to join
+  # @return [String] the resulting expanded path
   def self.path_to *pathsegs
-    begin
-      path = Pathname.new IMW.path_to_helper(*pathsegs)
-      path.absolute? ? File.expand_path(path) : path.to_s
-    rescue Exception => e
-      raise("Can't find path to '#{pathsegs}': #{e}");
-    end
+    path = Pathname.new IMW.path_to_helper(*pathsegs)
+    path.absolute? ? File.expand_path(path) : path.to_s
+  end
+
+  # Adds a symbolic path for expansion by +path_to+.
+  # 
+  #   IMW.add_path :foo, '~/whoa'
+  #   IMW.add_path :bar, :foo,   'baz'
+  #   IMW.path_to :bar
+  #   => '~/whoa/baz'
+  #
+  # @param [Symbol] sym the name of the path to store
+  # @param [Symbol, String] pathsegs the path segments to use to define the path to the name
+  # @return [String] the resulting path
+  def self.add_path sym, *pathsegs
+    IMW::PATHS[sym] = pathsegs.flatten
+    path_to[sym]
   end
 
-  private
+  # Removes a symbolic path for expansion by +path_to+.
+  #
+  # @param [Symbol] sym the stored path symbol to remove
+  def self.remove_path sym
+    IMW::PATHS.delete sym if IMW::PATHS.include? sym
+  end
+
+  protected
   def self.path_to_helper *pathsegs # :nodoc:
     # +path_to_helper+ handles the recursive calls for +path_to+.
     expanded = pathsegs.flatten.compact.map do |pathseg|
@@ -71,17 +143,4 @@ module IMW
     end
     File.join(*expanded)
   end
-  public
-
-  # Adds a symbolic path for expansion by +path_to+.
-  def self.add_path sym, *pathsegs
-    IMW::PATHS[sym] = pathsegs.flatten
-  end
-
-  # Removes a symbolic path for expansion by +path_to+.
-  def self.remove_path sym
-    IMW::PATHS.delete sym if IMW::PATHS.include? sym
-  end
 end
-
-# puts "#{File.basename(__FILE__)}: Your monkeywrench glows alternately dim then bright as you wander, suggesting to you which paths to take."

data/lib/imw/utils/version.rb

@@ -1,12 +1,11 @@
-# copied from activewarehouse-etl gem
-module IMWVersion #:nodoc:
+module IMW
   unless defined?(VERSION)
-  module VERSION #:nodoc:
-    MAJOR = 0
-    MINOR = 0
-    TINY  = 0
-
-    STRING = [MAJOR, MINOR, TINY].join('.')
-  end
+    module VERSION #:nodoc:
+      MAJOR = 0
+      MINOR = 0
+      TINY  = 0
+      
+      STRING = [MAJOR, MINOR, TINY].join('.')
+    end
   end
 end

data/lib/imw/utils.rb

@@ -1,24 +1,8 @@
-#
-# h2. lib/imw/utils.rb -- utility functions
-#
-# == About
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-
 require 'rubygems'
+require 'fileutils'
 require 'imw/utils/error'
 require 'imw/utils/log'
-require 'imw/utils/config'
 require 'imw/utils/paths'
 require 'imw/utils/misc'
-require 'imw/utils/components'
-require 'imw/utils/extensions/core'
-require 'fileutils'
-require 'pathname'
-
+require 'imw/utils/extensions'
 
-# puts "#{File.basename(__FILE__)}: Early economists thought they would measure the utility of an action in units of `utils'.  Really." # at bottom

data/lib/imw.rb

@@ -1,11 +1,6 @@
 require 'rubygems'
 require 'imw/boot'
 require 'imw/utils'
-require 'imw/dataset'
-require 'imw/repository'
-require 'imw/files'
-require 'imw/parsers'
-require 'imw/packagers'
 
 # The Infinite Monkeywrench (IMW) is a Ruby library for ripping,
 # extracting, parsing, munging, and packaging datasets.  It allows you
@@ -13,19 +8,99 @@ require 'imw/packagers'
 # transformations of data as a network of dependencies (a la Make or
 # Rake).
 #
-# On first reading of IMW examine the classes within the IMW::Files
-# module, all transparently instantiated when using IMW.open (instead
-# of File.open).  These classes do a lot of work to ensure that all
-# objects returned by IMW.open share methods (write, read, load, dump,
-# parse, compress, extract, &c.) while continuing to use existing
-# implementations of these concepts.  
+# IMW has a few central concepts: resources, datasets, workflows, and
+# repositories.
 #
-# Another entrace point is the <tt>IMW::Dataset</tt> class.  It
-# leverages Rake to craft workflows for transforming datasets.  IMW
-# encourages you to organize your data transformations in a step-wise
-# process, managed with dependencies.
+# Resources represent individual data resources like local files,
+# websites, databases, &c.  Resources are typically instantiated via
+# IMW.open, with IMW doing the work of figuring out what to return
+# based on the URI passed in.
 #
-# Utilities to help with one step in particular (ripping, parsing,
-# pacaking, &c.) are in their own directories.
+# Datasets represent collections of related data resources.  An
+# IMW::Dataset comes with a pre-defined (but customizable) workflow
+# that takes data resources through several steps: rip, parse, munge,
+# and package.  The workflow leverages Rake and so the various tasks
+# that are necessary to process the data till it is nice and pretty
+# can all be linked with dependencies.
+#
+# Repositories are collections of datasets and it is on these
+# collections that the +imw+ command line tool operates.
 module IMW
+  autoload :Resource,   'imw/resource'
+  autoload :Resources,  'imw/resources'
+  autoload :Repository, 'imw/repository'
+  autoload :Dataset,    'imw/dataset'
+  autoload :Transforms, 'imw/transforms'
+  autoload :Parsers,    'imw/parsers'
+
+  # Open a resource at the given +uri+.  The resource will
+  # automatically be extended by modules which make sense given the
+  # +uri+.
+  # 
+  # See the documentation for IMW::Resource and the various modules
+  # within IMW::Resources for more information and options.
+  #
+  # Passing in an IMW::Resource will simply return it.
+  #
+  # @param  [String, Addressable::URI, IMW::Resource] obj the URI to open
+  # @return [IMW::Resource] the resulting resource, property extended for the given URI
+  def self.open obj, options={}
+    return obj if obj.is_a?(IMW::Resource)
+    IMW::Resource.new(obj, options)
+  end
+
+  # Works the same way as IMW.open except opens the resource for
+  # writing.
+  #
+  # @param  [String, Addressable::URI] uri the URI to open
+  # @return [IMW::Resource] the resultng resource, properly extended for the given URI and opened for writing.
+  def self.open! uri, options={}
+    IMW::Resource.new(uri, options.merge(:mode => 'w'))
+  end
+
+  # The default repository in which to place datasets.  See the
+  # documentation for IMW::Repository for more information on how
+  # datasets and repositories fit together.
+  #
+  # @return [IMW::Repository] the default IMW repository
+  def self.repository
+    @@repository ||= IMW::Repository.new
+  end
+
+  # Create a dataset and put it in the default IMW repository.  Also
+  # yields the dataset so you can define its workflow
+  #
+  # IMW.dataset :my_dataset do
+  # 
+  #   # Define some paths we're going to use
+  #   add_path :raw_data,  :ripd, 'raw_data.csv'
+  #   add_path :fixd_data, :fixd, 'fixed_data.csv'
+  #
+  #   # Copy a file from a website to this dataset's +ripd+ directory.
+  #   rip do
+  #     IMW.open('http://mysite.com/data_archives/2010/03/03.csv').cp(path_to(:raw_data))
+  #   end
+  #
+  #   # Filter the raw data to those values which match some criterion defined by <tt>accept?</tt>
+  #   munge do
+  #     IMW.open(path_to(:raw_data)).map do |row|
+  #       row if accept?(row)
+  #     end.compact.dump(path_to(:fixd_data))
+  #   end
+  #
+  #   # Compress this new data
+  #   package do
+  #     IMW.open(path_to(:fixd_data)).compress.mv(path_to(:pkgd))
+  #   end
+  # end
+  #
+  # @param [Symbol, String] handle the handle to identify this dataset with
+  # @param [Hash]   options a hash of options (see IMW::Dataset)
+  # @return [IMW::Dataset] the new dataset
+  def self.dataset handle, options={}, &block
+    d = IMW::Dataset.new(handle, options)
+    d.instance_eval(&block) if block_given?
+    d
+  end
+
 end

data/spec/data/sample.csv

@@ -81,7 +81,7 @@ ID,Name,Genus,Species
 080,Tonkean Macaque,Macaca,tonkeana
 081,Heck's Macaque,Macaca,hecki
 082,Gorontalo Macaque,Macaca,nigrescens
-083,Celebes Crested Macaque or Black "Ape",Macaca,nigra
+083,Celebes Crested Macaque or Black Ape,Macaca,nigra
 084,Crab-eating Macaque or Long-tailed Macaque or Kera,Macaca,fascicularis
 085,Stump-tailed Macaque or Bear Macaque,Macaca,arctoides
 086,Rhesus Macaque,Macaca,mulatta

data/spec/data/sample.json

@@ -0,0 +1 @@
+{"monkeys":[{"monkey":{"name":"Gray-bellied Night Monkey","id":1,"genus":"Aotus","species":"lemurinus"}},{"monkey":{"name":"Panamanian Night Monkey","id":2,"genus":"Aotus","species":"zonalis"}},{"monkey":{"name":"Hern\u00e1ndez-Camacho's Night Monkey","id":3,"genus":"Aotus","species":"jorgehernandezi"}},{"monkey":{"name":"Gray-handed Night Monkey","id":4,"genus":"Aotus","species":"griseimembra"}},{"monkey":{"name":"Hershkovitz's Night Monkey","id":5,"genus":"Aotus","species":"hershkovitzi"}},{"monkey":{"name":"Brumback's Night Monkey","id":6,"genus":"Aotus","species":"brumbacki"}},{"monkey":{"name":"Three-striped Night Monkey","id":7,"genus":"Aotus","species":"trivirgatus"}},{"monkey":{"name":"Spix's Night Monkey","id":"008","genus":"Aotus","species":"vociferans"}},{"monkey":{"name":"Malaysian Lar Gibbon","id":"009","genus":"Hylobates","species":"lar lar"}},{"monkey":{"name":"Carpenter's Lar Gibbon","id":8,"genus":"Hylobates","species":"lar carpenteri"}},{"monkey":{"name":"Central Lar Gibbon","id":9,"genus":"Hylobates","species":"lar entelloides"}},{"monkey":{"name":"Sumatran Lar Gibbon","id":10,"genus":"Hylobates","species":"lar vestitus"}},{"monkey":{"name":"Yunnan Lar Gibbon","id":11,"genus":"Hylobates","species":"lar yunnanensis"}},{"monkey":{"name":"Mountain Agile Gibbon","id":12,"genus":"Hylobates","species":"agilis agilis"}},{"monkey":{"name":"Bornean White-bearded Gibbon","id":13,"genus":"Hylobates","species":"agilis albibarbis"}},{"monkey":{"name":"Lowland Agile Gibbon","id":14,"genus":"Hylobates","species":"agilis unko"}},{"monkey":{"name":"M\u00fcller's Gray Gibbon","id":15,"genus":"Hylobates","species":"muelleri muelleri"}},{"monkey":{"name":"Abbott's Gray Gibbon","id":"018","genus":"Hylobates","species":"muelleri abbotti"}},{"monkey":{"name":"Northern Gray Gibbon","id":"019","genus":"Hylobates","species":"muelleri funereus"}},{"monkey":{"name":"Black Tamarin","id":16,"genus":"Saguinas","species":"niger"}},{"monkey":{"name":"Black-mantled Tamarin","id":17,"genus":"Saguinas","species":"nigricollis"}},{"monkey":{"name":"Brown-mantled Tamarin","id":18,"genus":"Saguinas","species":"fuscicollis"}},{"monkey":{"name":"Cottontop Tamarin or Pinch\u00e9 Tamarin","id":19,"genus":"Saguinas","species":"oedipus"}},{"monkey":{"name":"Emperor Tamarin","id":20,"genus":"Saguinas","species":"imperator"}},{"monkey":{"name":"Geoffroy's Tamarin","id":21,"genus":"Saguinas","species":"geoffroyi"}},{"monkey":{"name":"Golden-mantled Tamarin","id":22,"genus":"Saguinas","species":"tripartitus"}},{"monkey":{"name":"Graells's Tamarin","id":23,"genus":"Saguinas","species":"graellsi"}},{"monkey":{"name":"Martins's Tamarin","id":"028","genus":"Saguinas","species":"martinsi"}},{"monkey":{"name":"Mottle-faced Tamarin","id":"029","genus":"Saguinas","species":"inustus"}},{"monkey":{"name":"Moustached Tamarin","id":24,"genus":"Saguinas","species":"mystax"}},{"monkey":{"name":"Pied Tamarin","id":25,"genus":"Saguinas","species":"bicolor"}},{"monkey":{"name":"Red-capped Tamarin","id":26,"genus":"Saguinas","species":"pileatus"}},{"monkey":{"name":"Red-handed Tamarin","id":27,"genus":"Saguinas","species":"midas"}},{"monkey":{"name":"White-footed Tamarin","id":28,"genus":"Saguinas","species":"leucopus"}},{"monkey":{"name":"White-lipped Tamarin","id":29,"genus":"Saguinas","species":"labiatus"}},{"monkey":{"name":"White-mantled Tamarin","id":30,"genus":"Saguinas","species":"melanoleucus"}},{"monkey":{"name":"Allen's Swamp Monkey","id":31,"genus":"Allenopithecus","species":"nigroviridis"}},{"monkey":{"name":"Angolan Talapoin","id":"038","genus":"Miopithecus","species":"talapoin"}},{"monkey":{"name":"Gabon Talapoin","id":"039","genus":"Miopithecus","species":"ogouensis"}},{"monkey":{"name":"Patas Monkey","id":32,"genus":"Erythrocebus","species":"patas"}},{"monkey":{"name":"Green Monkey","id":33,"genus":"Chlorocebus","species":"sabaeus"}},{"monkey":{"name":"Grivet","id":34,"genus":"Chlorocebus","species":"aethiops"}},{"monkey":{"name":"Bale Mountains Vervet","id":35,"genus":"Chlorocebus","species":"djamdjamensis"}},{"monkey":{"name":"Tantalus Monkey","id":36,"genus":"Chlorocebus","species":"tantalus"}},{"monkey":{"name":"Vervet Monkey","id":37,"genus":"Chlorocebus","species":"pygerythrus"}},{"monkey":{"name":"Malbrouck","id":38,"genus":"Chlorocebus","species":"cynosuros"}},{"monkey":{"name":"Dryas Monkey or Salongo Monkey","id":39,"genus":"Cercopithecus","species":"dryas"}},{"monkey":{"name":"Diana Monkey","id":"048","genus":"Cercopithecus","species":"diana"}},{"monkey":{"name":"Roloway Monkey","id":"049","genus":"Cercopithecus","species":"roloway"}},{"monkey":{"name":"Greater Spot-nosed Monkey","id":40,"genus":"Cercopithecus","species":"nictitans"}},{"monkey":{"name":"Blue Monkey","id":41,"genus":"Cercopithecus","species":"mitis"}},{"monkey":{"name":"Silver Monkey","id":42,"genus":"Cercopithecus","species":"doggetti"}},{"monkey":{"name":"Golden Monkey","id":43,"genus":"Cercopithecus","species":"kandti"}},{"monkey":{"name":"Sykes's Monkey","id":44,"genus":"Cercopithecus","species":"albogularis"}},{"monkey":{"name":"Mona Monkey","id":45,"genus":"Cercopithecus","species":"mona"}},{"monkey":{"name":"Campbell's Mona Monkey","id":46,"genus":"Cercopithecus","species":"campbelli"}},{"monkey":{"name":"Lowe's Mona Monkey","id":47,"genus":"Cercopithecus","species":"lowei"}},{"monkey":{"name":"Crested Mona Monkey","id":"058","genus":"Cercopithecus","species":"pogonias"}},{"monkey":{"name":"Wolf's Mona Monkey","id":"059","genus":"Cercopithecus","species":"wolfi"}},{"monkey":{"name":"Dent's Mona Monkey","id":48,"genus":"Cercopithecus","species":"denti"}},{"monkey":{"name":"Lesser Spot-nosed Monkey","id":49,"genus":"Cercopithecus","species":"petaurista"}},{"monkey":{"name":"White-throated Guenon","id":50,"genus":"Cercopithecus","species":"erythrogaster"}},{"monkey":{"name":"Sclater's Guenon","id":51,"genus":"Cercopithecus","species":"sclateri"}},{"monkey":{"name":"Red-eared Guenon","id":52,"genus":"Cercopithecus","species":"erythrotis"}},{"monkey":{"name":"Moustached Guenon","id":53,"genus":"Cercopithecus","species":"cephus"}},{"monkey":{"name":"Red-tailed Monkey","id":54,"genus":"Cercopithecus","species":"ascanius"}},{"monkey":{"name":"L'Hoest's Monkey","id":55,"genus":"Cercopithecus","species":"lhoesti"}},{"monkey":{"name":"Preuss's Monkey","id":"068","genus":"Cercopithecus","species":"preussi"}},{"monkey":{"name":"Sun-tailed Monkey","id":"069","genus":"Cercopithecus","species":"solatus"}},{"monkey":{"name":"Hamlyn's Monkey","id":56,"genus":"Cercopithecus","species":"hamlyni"}},{"monkey":{"name":"De Brazza's Monkey","id":57,"genus":"Cercopithecus","species":"neglectus"}},{"monkey":{"name":"Barbary Macaque","id":58,"genus":"Macaca","species":"sylvanus"}},{"monkey":{"name":"Lion-tailed Macaque","id":59,"genus":"Macaca","species":"silenus"}},{"monkey":{"name":"Southern Pig-tailed Macaque or Beruk","id":60,"genus":"Macaca","species":"nemestrina"}},{"monkey":{"name":"Northern Pig-tailed Macaque","id":61,"genus":"Macaca","species":"leonina"}},{"monkey":{"name":"Pagai Island Macaque or Bokkoi","id":62,"genus":"Macaca","species":"pagensis"}},{"monkey":{"name":"Siberut Macaque","id":63,"genus":"Macaca","species":"siberu"}},{"monkey":{"name":"Moor Macaque","id":"078","genus":"Macaca","species":"maura"}},{"monkey":{"name":"Booted Macaque","id":"079","genus":"Macaca","species":"ochreata"}},{"monkey":{"name":"Tonkean Macaque","id":"080","genus":"Macaca","species":"tonkeana"}},{"monkey":{"name":"Heck's Macaque","id":"081","genus":"Macaca","species":"hecki"}},{"monkey":{"name":"Gorontalo Macaque","id":"082","genus":"Macaca","species":"nigrescens"}},{"monkey":{"name":"Celebes Crested Macaque or Black Ape","id":"083","genus":"Macaca","species":"nigra"}},{"monkey":{"name":"Crab-eating Macaque or Long-tailed Macaque or Kera","id":"084","genus":"Macaca","species":"fascicularis"}},{"monkey":{"name":"Stump-tailed Macaque or Bear Macaque","id":"085","genus":"Macaca","species":"arctoides"}},{"monkey":{"name":"Rhesus Macaque","id":"086","genus":"Macaca","species":"mulatta"}},{"monkey":{"name":"Formosan Rock Macaque","id":"087","genus":"Macaca","species":"cyclopis"}},{"monkey":{"name":"Japanese Macaque","id":"088","genus":"Macaca","species":"fuscata"}},{"monkey":{"name":"Toque Macaque","id":"089","genus":"Macaca","species":"sinica"}},{"monkey":{"name":"Bonnet Macaque","id":"090","genus":"Macaca","species":"radiata"}},{"monkey":{"name":"Assam Macaque","id":"091","genus":"Macaca","species":"assamensis"}},{"monkey":{"name":"Tibetan Macaque or Milne-Edwards' Macaque","id":"092","genus":"Macaca","species":"thibetana"}},{"monkey":{"name":"Arunachal Macaque or Munzala","id":"093","genus":"Macaca","species":"munzala"}},{"monkey":{"name":"Grey-cheeked Mangabey","id":"094","genus":"Lophocebus","species":"albigena"}},{"monkey":{"name":"Black Crested Mangabey","id":"095","genus":"Lophocebus","species":"aterrimus"}},{"monkey":{"name":"Opdenbosch's Mangabey","id":"096","genus":"Lophocebus","species":"opdenboschi"}},{"monkey":{"name":"Uganda Mangabey","id":"097","genus":"Lophocebus","species":"ugandae"}},{"monkey":{"name":"Johnston's Mangabey","id":"098","genus":"Lophocebus","species":"johnstoni"}},{"monkey":{"name":"Osman Hill's Mangabey","id":"099","genus":"Lophocebus","species":"osmani"}},{"monkey":{"name":"Kipunji","id":100,"genus":"Rungwecebus","species":"kipunji"}},{"monkey":{"name":"Hamadryas Baboon","id":101,"genus":"Papio","species":"hamadryas"}},{"monkey":{"name":"Guinea Baboon","id":102,"genus":"Papio","species":"papio"}},{"monkey":{"name":"Olive Baboon","id":103,"genus":"Papio","species":"anubis"}},{"monkey":{"name":"Yellow Baboon","id":104,"genus":"Papio","species":"cynocephalus"}},{"monkey":{"name":"Chacma Baboon","id":105,"genus":"Papio","species":"ursinus"}},{"monkey":{"name":"Gelada","id":106,"genus":"Theropithecus","species":"gelada"}},{"monkey":{"name":"Sooty Mangabey","id":107,"genus":"Cercocebus","species":"atys"}},{"monkey":{"name":"Collared Mangabey","id":108,"genus":"Cercocebus","species":"torquatus"}},{"monkey":{"name":"Agile Mangabey","id":109,"genus":"Cercocebus","species":"agilis"}},{"monkey":{"name":"Golden-bellied Mangabey","id":110,"genus":"Cercocebus","species":"chrysogaster"}},{"monkey":{"name":"Tana River Mangabey","id":111,"genus":"Cercocebus","species":"galeritus"}},{"monkey":{"name":"Sanje Mangabey","id":112,"genus":"Cercocebus","species":"sanjei"}},{"monkey":{"name":"Mandrill","id":113,"genus":"Mandrillus","species":"sphinx"}},{"monkey":{"name":"Drill","id":114,"genus":"Mandrillus","species":"leucophaeus"}},{"monkey":{"name":"Black Colobus","id":115,"genus":"Colobus","species":"satanas"}},{"monkey":{"name":"Angola Colobus","id":116,"genus":"Colobus","species":"angolensis"}},{"monkey":{"name":"King Colobus","id":117,"genus":"Colobus","species":"polykomos"}},{"monkey":{"name":"Ursine Colobus","id":118,"genus":"Colobus","species":"vellerosus"}},{"monkey":{"name":"Mantled Guereza","id":119,"genus":"Colobus","species":"guereza"}},{"monkey":{"name":"Western Red Colobus","id":120,"genus":"Piliocolobus","species":"badius"}},{"monkey":{"name":"Pennant's Colobus","id":121,"genus":"Piliocolobus","species":"pennantii"}},{"monkey":{"name":"Preuss's Red Colobus","id":122,"genus":"Piliocolobus","species":"preussi"}},{"monkey":{"name":"Thollon's Red Colobus","id":123,"genus":"Piliocolobus","species":"tholloni"}},{"monkey":{"name":"Central African Red Colobus","id":124,"genus":"Piliocolobus","species":"foai"}},{"monkey":{"name":"Ugandan Red Colobus","id":125,"genus":"Piliocolobus","species":"tephrosceles"}},{"monkey":{"name":"Uzungwa Red Colobus","id":126,"genus":"Piliocolobus","species":"gordonorum"}},{"monkey":{"name":"Zanzibar Red Colobus","id":127,"genus":"Piliocolobus","species":"kirkii"}},{"monkey":{"name":"Tana River Red Colobus","id":128,"genus":"Piliocolobus","species":"rufomitratus"}},{"monkey":{"name":"Olive Colobus","id":129,"genus":"Procolobus","species":"verus"}},{"monkey":{"name":"Maroon Leaf Monkey","id":130,"genus":"Presbytis","species":"rubicunda"}}]}

data/spec/data/sample.tsv

@@ -81,7 +81,7 @@ ID	Name	Genus	Species
 080	Tonkean Macaque	Macaca	tonkeana
 081	Heck's Macaque	Macaca	hecki
 082	Gorontalo Macaque	Macaca	nigrescens
-083	Celebes Crested Macaque or Black "Ape"	Macaca	nigra
+083	Celebes Crested Macaque or Black Ape	Macaca	nigra
 084	Crab-eating Macaque or Long-tailed Macaque or Kera	Macaca	fascicularis
 085	Stump-tailed Macaque or Bear Macaque	Macaca	arctoides
 086	Rhesus Macaque	Macaca	mulatta

data/spec/data/sample.txt

@@ -81,7 +81,7 @@ ID,Name,Genus,Species
 080,Tonkean Macaque,Macaca,tonkeana
 081,Heck's Macaque,Macaca,hecki
 082,Gorontalo Macaque,Macaca,nigrescens
-083,Celebes Crested Macaque or Black "Ape",Macaca,nigra
+083,Celebes Crested Macaque or Black Ape,Macaca,nigra
 084,Crab-eating Macaque or Long-tailed Macaque or Kera,Macaca,fascicularis
 085,Stump-tailed Macaque or Bear Macaque,Macaca,arctoides
 086,Rhesus Macaque,Macaca,mulatta

data/spec/data/sample.xml

@@ -412,7 +412,7 @@
   <monkey id="083">
     <genus>Macaca</genus>
     <species>nigra</species>
-    <name>Celebes Crested Macaque or Black "Ape"</name>
+    <name>Celebes Crested Macaque or Black Ape</name>
   </monkey>
   <monkey id="084">
     <genus>Macaca</genus>

data/spec/data/sample.yaml

@@ -412,7 +412,7 @@ monkeys:
     species: nigrescens
 - monkey:
     id: 083
-    name: Celebes Crested Macaque or Black "Ape"
+    name: Celebes Crested Macaque or Black Ape
     genus: Macaca
     species: nigra
 - monkey:

data/spec/imw/dataset/paths_spec.rb

@@ -0,0 +1,32 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+require File.dirname(__FILE__) + "/../utils/shared_paths_spec"
+
+describe IMW::Dataset do
+
+  describe 'setting default paths' do
+
+    before do
+      @dataset = IMW::Dataset.new(:testing, :root => IMWTest::TMP_DIR)
+    end
+
+    it "should set its root path to the value given" do
+      @dataset.path_to(:root).should == IMWTest::TMP_DIR
+    end
+
+    it "should set paths for each workflow dir" do
+      @dataset.workflow_dirs.each do |dir|
+        @dataset.path_to(dir).should == File.join(IMWTest::TMP_DIR, dir.to_s)
+      end
+    end
+
+    before do
+      @path_manager = @dataset
+    end
+    it_should_behave_like "an object that manages paths"
+    
+  end
+end
+
+
+
+

data/spec/imw/dataset/workflow_spec.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+require 'imw/dataset/workflow'
+describe IMW::Workflow do
+
+  before do
+    @dataset = IMW::Dataset.new :testing
+  end
+
+  it "should dynamically define methods for each workflow step" do
+    @dataset.workflow_steps.each do |step|
+      @dataset.respond_to?(step).should be_true
+    end
+  end
+
+  describe "initializing workflow" do
+    it "should not make any directories if no tasks are invoked" do
+      @dataset.path_to(:root).should_not contain(*@dataset.workflow_dirs.map(&:to_s))
+    end
+
+    it "should only make directories once a task is invoked" do
+      @dataset[:initialize].invoke
+      @dataset.path_to(:root).should contain(*@dataset.workflow_dirs.map(&:to_s))
+    end
+  end
+
+  describe "cleaning workflow directories" do
+    it "should clean without error even if there's nothing to clean" do
+      @dataset[:clean].invoke
+      @dataset.path_to(:root).should_not contain(*@dataset.workflow_dirs.map(&:to_s))
+    end
+
+    it "should remove workflow directories when invoked" do
+      @dataset[:initialize].invoke
+      IMWTest::Random.file(@dataset.path_to(:ripd, 'foobar.txt')) # put a file in
+      @dataset[:clean].invoke
+      @dataset.path_to(:root).should_not contain(*@dataset.workflow_dirs.map(&:to_s))
+    end
+  end
+
+end
+