imw 0.1.1 → 0.2.0

data/.gitignore

@@ -12,4 +12,7 @@ TAGS
 tmp/*
 *.tmproj
 pkg/*
-imw.gemspec
+*gemspec
+tags
+.yardoc/*
+*/.yardoc/*

data/Rakefile

@@ -18,3 +18,13 @@ begin
 rescue LoadError
   puts "Jeweler (or a dependency) not available.  Install it with: sudo gem install jeweler"
 end
+
+desc "Build tags"
+task :tags do
+  system "etags -R bin etc examples lib spec"
+end
+
+desc "Build docs"
+task :docs do
+  system "yardoc"
+end

data/TODO

@@ -0,0 +1,18 @@
+lookup basic yarddoc style (@params, etc) -- do a high-level description
+
+learn how to run specs
+write a spec that fails on the old code and passes on the new
+
+convert all references to URI to be Addressable::URI
+don't use URI.parse, use Addressable::URI.heuristic_parse (eg in files/*)
+make basicfile methods delegate to its uri
+
+tmpdir should use the actual system tmpdir libs (eg in archiver)
+move config over to configliere
+
+
+
+
+------ WANT PONY -----
+
+might be nice to learn the delegate pattern

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.2.0

data/bin/imw

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 $:.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'imw/runner'
-exit IMW::Runner.new(*ARGV).run!
+exit IMW::Runner.new(*ARGV.dup).run!
 

data/etc/imwrc.rb

@@ -21,56 +21,6 @@
 # Copyright:: Copyright (c) 2008 infochimps.org
 # License::   GPL 3.0
 # Website::   http://infinitemonkeywrench.org/
-#
 
 module IMW
-  PATHS = {
-    :home         => ENV['HOME'],
-    :data_root    => "/var/lib/imw",
-    :log_root     => "/var/log/imw",
-    :scripts_root => "/usr/share/imw",
-    :tmp_root     => "/tmp/imw",
-
-    # the imw library
-    :imw_root  => File.expand_path(File.dirname(__FILE__) + "/.."),
-    :imw_bin   => [:imw_root, 'bin'],
-    :imw_etc   => [:imw_root, 'etc'],
-    :imw_lib   => [:imw_root, 'lib'],
-
-    # workflow
-    :ripd_root  => [:data_root, 'ripd'],
-    :peeld_root => [:data_root, 'peeld'],
-    :mungd_root => [:data_root, 'mungd'],
-    :temp_root  => [:data_root, 'temp'],
-    :fixd_root  => [:data_root, 'fixd'],
-    :pkgd_root  => [:data_root, 'pkgd']
-  }
-
-  # Default time format.
-  STRFTIME_FORMAT = "%Y%m%d-%H%M%S" unless defined? STRFTIME_FORMAT
-
-  # Paths to external programs used by IMW.
-  EXTERNAL_PROGRAMS = {
-    :tar => "tar",
-    :rar => "rar",
-    :zip => "zip",
-    :unzip => "unzip",
-    :gzip => "gzip",
-    :bzip2 => "bzip2",
-    :wget => "wget"
-  } unless defined? ::IMW::EXTERNAL_PROGRAMS
-
-  module Files
-    # Regular expressions which match pathnames to the name of the
-    # appropriate IMW::Files class.
-    #
-    # File class names should be stripped of the leading
-    # <tt>IMW::Files</tt> prefix, i.e. - the file object
-    # <tt>IMW::Files::Bz2</tt> should be referenced by the string
-    # <tt>"Bz2"</tt>.
-    FILE_REGEXPS = [] unless defined? ::IMW::Files::FILE_REGEXPS
-  end
-
 end
-
-

data/examples/dataset.rb

@@ -0,0 +1,12 @@
+require 'imw'
+dataset = IMW::Dataset.new :handle => 'test'
+
+dataset.rip do
+  IMW.open("http://path/to/somre/resource.html").cp(dataset.path_to(:ripd), 'original_data.html')
+end
+
+dataset.parse do
+  #...
+end
+
+

data/lib/imw/boot.rb

@@ -1,4 +1,36 @@
+require 'imw/utils/extensions/hash'
+
 module IMW
+
+  # IMW looks for configuration settings in the following places, in
+  # order of increasing precedence:
+  #
+  #   1. Settings defined directly in this file.
+  #
+  #   2. From the <tt>etc/imwrc</tt> file in the IMW root directory.
+  #
+  #   3. From the <tt>.imwrc</tt> file in the user's home directory (the
+  #      filename can be changed; see
+  #      <tt>IMW::Config::USER_CONFIG_FILE_BASENAME</tt>).
+  #
+  #   4. From the file defined by the environment variable +IMWRC+ (the
+  #      value can be changed; see
+  #      <tt>IMW::Config::USER_CONFIG_FILE_ENV_VARIABLE</tt>
+  #
+  # Settings not found in one configuration location will be searched
+  # for in locations of lesser precedence.
+  #
+  # *Note:* configuration files are plain Ruby code that will be directly
+  # evaluated.
+  #
+  # Relevant settings include
+  #
+  # * interfaces with external programs (+tar+, +wget+, &c.)
+  # * paths to directories where IMW reads/writes files
+  # * correspondences between file extensions and IMW file classes
+  #
+  # For more detailed information, see the default configuration file,
+  # <tt>etc/imwrc</tt>.
   module Config
 
     # Root of the IMW source base.
@@ -9,11 +41,12 @@ module IMW
     #
     # User configuration file
     #
-    # By default, the file ~/.imwrc (.imwrc, in your home directory -- note no .rb extension)
-    # is sourced at top level.  If the $IMWRC environment variable is set,
-    # that file will be sourced instead.
+    # By default, the file ~/.imwrc (.imwrc, in your home directory --
+    # note no .rb extension) is sourced at top level.  If the $IMWRC
+    # environment variable is set, that file will be sourced instead.
     #
-    # Any code within this file will override settings in IMW_ROOT/etc/imwrc.rb
+    # Any code within this file will override settings in
+    # /etc/imwrc.rb which itself overrides IMW_ROOT/etc/imwrc.rb
     #
     USER_CONFIG_FILE = File.join(ENV['HOME'] || '', '.imwrc')
     # Environment variable to override user configuration file location.
@@ -22,16 +55,29 @@ module IMW
       File.expand_path(ENV[ENV_CONFIG_FILE] || USER_CONFIG_FILE)
     end
 
-    # System-level config file
-    SITE_CONFIG_FILE = "etc/imwrc.rb"
+    # Path to site-wide config file (overwrites IMW defaults but
+    # overridden by user defaults).
+    SITE_CONFIG_FILE = "/etc/imwrc.rb"
     def self.site_config_file # :nodoc:
-      File.join(imw_root, SITE_CONFIG_FILE)
+      SITE_CONFIG_FILE
     end
 
+    def self.default_config_file # :nodoc:
+      File.join(imw_root, "etc/imwrc.rb")
+    end
+    
     # Source the config files
     def self.load_config
-      require site_config_file
-      load    user_config_file   if File.exist? user_config_file
+      if File.exist?(user_config_file)
+        load user_config_file
+      end
+
+      if File.exist?(site_config_file)
+        load site_config_file
+      end
+
+      load default_config_file
+      
     end
   end
 end

data/lib/imw/dataset/paths.rb

@@ -1,32 +1,24 @@
 module IMW
-
   class Dataset
     include IMW::Paths
 
-    # A dataset keeps track of its own collection of paths just like
-    # IMW itself.  When an IMW::Dataset is instantiated in a script,
-    # that script's directory becomes the dataset's +self+ path and
-    # the default workflow directories (see IMW::Workflow) are created
-    # within this directory.
+    protected
+    # Sets paths to the workflow directories for this dataset (+ripd+,
+    # +rawd+, +fixd+, +pkgd+) as well as the following paths:
     #
-    # You can change a dataset's paths the same way you can change
-    # IMW's paths; calling +add_path+ and +remove_path+ on the
-    # dataset.
+    # script::
+    #   The path to the file the dataset was initialized in.
     #
-    # To customize this behavior for all future datasets, created a
-    # subclass of IMW::Dataset and override the +set_paths+ method.
-    def paths
-      @paths
-    end
-    
-    protected
-    # Sets the roots of various paths relative to this dataset.
-    def set_root_paths
-      @paths = {}
-      add_path :script, File.expand_path(eval('__FILE__'))      
-      add_path :self,   File.dirname(path_to(:script))
-      IMW::Workflow::DIRS.each do |dir|
-        add_path dir, :self, dir.to_s
+    # root::
+    #   The parent directory of the file the dataset was initialized
+    #   in or the value of the <tt>:root</tt> key in
+    #   IMW::Dataset#options
+    #
+    def set_default_paths
+      add_path :script, File.expand_path(eval('__FILE__'))
+      add_path :root,   options[:root] || File.dirname(path_to(:script))
+      workflow_dirs.each do |dir|
+        add_path dir, :root, dir.to_s
       end
     end
 
@@ -34,5 +26,4 @@ module IMW
     def set_paths
     end
   end
-    
 end

data/lib/imw/dataset/workflow.rb

@@ -1,42 +1,62 @@
-require 'imw/dataset/task'
 require 'ostruct'
+require 'rake'
 
 module IMW
 
-  # IMW encourages you to view a data transformation as a network of
-  # dependencies.  By default, IMW defines five main steps:
+  # An IMW version of Rake::Task
+  Task             = Class.new(Rake::Task)
+
+  # An IMW subclass of Rake:FileTask
+  FileTask         = Class.new(Rake::FileTask)
+
+  # An IMW subclass of Rake::FileCreationTask
+  FileCreationTask = Class.new(Rake::FileCreationTask)
+
+  # IMW encourages you to view a data transformation as a series of
+  # interdependent steps.
   #
-  # rip::
-  #   Obtain data via HTTP, FTP, SCP, RSYNC, database query, &c.
+  # By default, IMW defines four main steps in such a transformation:
+  # +rip+, +parse+, +fix+, and +package+.
+  #
+  # Each step is associated with a directory on disk in which it keeps
+  # its files: +ripd+, +prsd+, +fixd+, and +pkgd+.
   #
-  # extract::
-  #   Extract data from its ripped form to a form which can be
-  #   parsed.
+  # The steps are:
+  #
+  # rip::
+  #   Obtain data via HTTP, FTP, SCP, RSYNC, database query, &c and
+  #   store the results in +ripd+.
   #
   # parse::
-  #   Parse data into a structured form.
+  #   Parse data into a structured form using a library (JSON, YAML,
+  #   &c.) or using your own parser (XML, flat files, &c.) and store
+  #   the results in +prsd+
   #
-  # munge::
+  # fix::
   #   Combine, filter, reconcile, and transform already structured
-  #   data into a desired form.
+  #   data into a desired form and store the results in +fixd+.
   #
   # package::
   #   Archive, compress, and deliver data in its final form to some
-  #   location (HTTP, FTP, SCP, RSYNC, S3, EBS, &c.).
+  #   location (HTTP, FTP, SCP, RSYNC, S3, EBS, &c.), optionally
+  #   storing the ouptut in +pkgd+.
   #
   # Each step depends upon the one before it.  The steps are blank by
   # default so there's no need to write code for steps you don't need
-  # to use.
+  # to use.  You can also define your own steps (using +task+ just
+  # like in Rake) and hook them into these pre-defined steps (or
+  # not...).
   #
-  # Each step corresponds to a named directory in IMW::Workflow::DIRS.
+  # A dataset also has an <tt>:initialize</tt> task (which by default
+  # just creates the directories for these steps) which you can use to
+  # hook in your own initialization tasks by making it depend on them.
+  #
+  # A subclass of IMW::Dataset can customize how tasks are defined by
+  # overriding +define_workflow_tasks+, among other methods, and
+  # introduce new tasks by overriding +define_tasks+.
   module Workflow
 
-    # The <tt>Rake::TaskManager</tt> module allows the
-    # <tt>IMW::Dataset</tt> class to leverage the functionality of the
-    # Rake[http://rake.rubyforge.org/] library to manage tasks
-    # associated with the processing of this dataset.
     include Rake::TaskManager
-
     # Default options passed to <tt>Rake</tt>.  Any class including
     # the <tt>Rake::TaskManager</tt> module must define a constant by
     # this name.
@@ -45,51 +65,77 @@ module IMW
       :trace   => false,
       :verbose => false
     }
+    
+    # Return a new (or existing) <tt>IMW::Task</tt> with the given
+    # +name+.  Dependencies can be declared and a block passed in just
+    # as in Rake.
+    #
+    # @param [Hash, Symbol, String] deps the name of the task (if a
+    # Symbol or String) or the name of the task mapped to an Array of
+    # dependencies (if a Hash)
+    #
+    # @return [IMW::Task] the task
+    def task deps, &block
+      self.define_task IMW::Task, deps, &block
+    end
+
+    # Return a new (or existing) <tt>IMW::FileTask</tt> with the given
+    # +path+.  Dependencies can be declared and a block passed in just
+    # as in Rake.
+    #
+    # @param [String, IMW::Resource] path the path to the file
+    # @return [IMW::FileTask] the task
+    def file path, &block
+      path = path.respond_to?(:path) ? path.path : path
+      self.define_task IMW::FileTask, path, &block
+    end
+
+    # Return a new (or existing) <tt>IMW::FileCreationTask</tt> with the given
+    # +path+.  Dependencies can be declared and a block passed in just
+    # as in Rake.
+    #
+    # @param [String, IMW::Resource] path the path to the file
+    # @return [IMW::FileCreationTask] the task
+    def file_create path, &block
+      path = path.respond_to?(:path) ? path.path : path
+      self.define_task IMW::FileCreationTask, path, &block
+    end
+
+    # Override this method to define default tasks for a subclass of
+    # IMW::Dataset.
+    def define_tasks
+    end
 
     # The standard IMW workflow steps.
-    STEPS = [:rip,  :extract, :parse, :munge, :package]
+    #
+    # @return [Array] the workflow step names
+    def workflow_steps
+      [:rip,  :parse, :fix, :package]
+    end
 
     # The steps of the IMW workflow each correspond to a directory in
     # which it is customary that they deposit their files <em>once
     # they are finished processing</em> (so ripped files wind up in
     # the +ripd+ directory, packaged files in the +pkgd+ directory,
     # and so on).
-    DIRS  = [:ripd, :xtrd,    :prsd,  :mungd, :pkgd   ]
-
-    # Each workflow step can be configured to take default actions,
-    # each action being a proc in the array for the step in this hash.
     #
-    # This allows classes which include IMW::Workflow to use class
-    # methods named after each step (+rip+, +parse+, &c.) to directly
-    # define tasks.
-    STEPS_TASKS = returning({}) do |steps_procs|
-      STEPS.each do |step|
-        steps_procs[step] = []
-      end
+    # @return [Array] the workflow directory names
+    def workflow_dirs
+      [:ripd, :rawd,  :fixd, :pkgd]
     end
 
     protected
-    def self.included klass
-      STEPS.each do |step|
-        klass.class_eval <<EOF
-def self.#{step}(deps=nil, &block)
-  STEPS_TASKS[:#{step}] << [deps, block]
-end
-EOF
-      end
-
-      
-    end
-
-    def define_workflow_task deps, comment
+    
+    # Convenience method for defining tasks for this workflow.
+    #
+    # @param [Hash, Symbol, String] deps the name of the task (if a
+    # Symbol or String) or the name of the task mapped to an Array of
+    # dependencies (if a Hash)
+    # @param [String] comment the comment to associate to the task
+    # @return [IMW::Task] the task
+    def define_workflow_task deps, comment, &block
       @last_description = comment
-      define_task(IMW::Task, deps)
-      step = deps.respond_to?(:keys) ? deps.keys.first : deps
-      STEPS_TASKS[step].each do |deps, block|
-        self[step].enhance(deps) do
-          self.instance_eval(&block)
-        end
-      end
+      define_task(IMW::Task, deps, &block)
     end
 
     # Create all the instance variables required by Rake::TaskManager
@@ -100,43 +146,56 @@ EOF
       @scope = Array.new
       @last_description = nil
       @options = OpenStruct.new(DEFAULT_OPTIONS)
-      define_create_directories_task
-      define_workflow_tasks      
-      define_destroy_task
+      define_initialize_task
+      define_workflow_tasks
+      define_workflow_task_methods
+      define_clean_task
+      define_tasks
     end
 
-    # Creates a task <tt>:create_directories</tt> to create the
-    # directory structure for this dataset.
-    def define_create_directories_task
-      @last_description = "Creates workflow directories for this dataset."
-      define_task(IMW::Task, {:create_directories => []}) do
-        DIRS.each do |dir|
+    # Defines the <tt>:initialize</tt> task.  The only other task
+    # hooked into <tt>:initialize</tt> is the
+    # <tt>:create_workflow_dirs</tt> task which creates the workflow
+    # directories for this dataset.
+    def define_initialize_task
+      define_workflow_task({:create_directories => []}, "Creates workflow directories for this dataset.") do
+        workflow_dirs.each do |dir|
           FileUtils.mkdir_p(path_to(dir)) unless File.exist?(path_to(dir))
         end
       end
+      define_workflow_task({ :initialize => [:create_directories] }, "Initialize this dataset.")
     end
 
-    # Creates a task <tt>:destroy</tt> which removes dataset's
+    # Creates a task <tt>:clean</tt> which removes dataset's
     # workflow directories.
-    def define_destroy_task
-      @last_description = "Get rid of all traces of this dataset."
-      define_task(IMW::Task, :destroy => [:create_directories]) do
-        DIRS.each do |dir|
-          FileUtils.rm_rf(path_to(dir))
+    def define_clean_task
+      define_workflow_task :clean, "Remove the workflow directories for this dataset." do
+        workflow_dirs.each do |dir|
+          FileUtils.rm_rf(path_to(dir)) if File.exist?(path_to(dir))
         end
       end
     end
 
-    # Creates the task dependency chain <tt>:package => :munge =>
-    # :parse => :extract => :rip => :initialize</tt> of the
+    # Creates the task dependency chain <tt>:package => :fix =>
+    # :parse => :rip => :initialize</tt> of the
     # IMW::Workflow.
     def define_workflow_tasks
       define_workflow_task({:rip     => [:create_directories]}, "Obtain data from some source."           )
-      define_workflow_task({:extract => [:rip]},                "Extract data so it's ready to parse."    )
-      define_workflow_task({:parse   => [:extract]},            "Parse data into a structured form."      )
-      define_workflow_task({:munge   => [:parse]},              "Munge structured data into desired form.")
-      define_workflow_task({:package => [:munge]},              "Package dataset in final form."          )
+      define_workflow_task({:parse   => [:rip]},                "Parse data into a structured form."      )
+      define_workflow_task({:fix     => [:parse]},              "Munge parsed data into desired form."    )
+      define_workflow_task({:package => [:fix]},                "Package dataset in final form."          )
     end
 
+    # Dynamically define methods for each of the workflow steps which
+    # act as shorcuts for accessing the corresponding tasks.
+    def define_workflow_task_methods
+      workflow_steps.each do |step|
+        self.class.class_eval <<RUBY
+          def #{step} deps, &block
+            self[step].enhance(step => deps, &block)
+          end
+RUBY
+      end
+    end
   end
 end