imw 0.2.18 → 0.3.0

data/lib/imw/compressed_files.rb

@@ -1,94 +0,0 @@
-module IMW
-
-  # Contains modules which define the behavior of compressed files.
-  module CompressedFiles
-    autoload :Bz2,          'imw/compressed_files/bz2'
-    autoload :Gz,           'imw/compressed_files/gz'
-    autoload :Compressible, 'imw/compressed_files/compressible'
-
-    # Handlers which include modules for compressed file formats as
-    # well as the IMW::CompressedFiles::Compressible module for
-    # compressing regular files.
-    HANDLERS = [
-                ["CompressedFiles::Compressible", Proc.new { |r| r.is_local? && r.is_file? && r.path != /\.(bz2|gz|tgz|tbz2)$/i                         } ],
-                ["CompressedFiles::Gz",           Proc.new { |r| r.is_local? && r.path =~ /\.gz$/i  && r.path !~ /\.tar\.gz$/i  && r.path !~ /\.tgz$/i  } ],
-                ["CompressedFiles::Bz2",          Proc.new { |r| r.is_local? && r.path =~ /\.bz2$/i && r.path !~ /\.tar\.bz2$/i && r.path !~ /\.tbz2$/i } ]    
-               ]
-
-    # Defines methods for decompressing a compressed file.  This
-    # module isn't used to directly extend an IMW::Resource --
-    # instead, format specific modules (e.g. -
-    # IMW::Resources::CompressedFiles::Bz2) include this module and
-    # further define the command-line flags &c. needed to make
-    # everything work.
-    module Base
-
-      attr_accessor :compression_settings
-
-      # Is this file compressed?
-      #
-      # @return [true, false]
-      def is_compressed?
-        true
-      end
-
-      # Can this file be compressed?
-      #
-      # @return [true, false]
-      def is_compressible?
-        false
-      end
-
-      # The basename of this resource after it is decompressed
-      #
-      #   IMW::Resource.new('/path/to/my_file.txt.bz2').decompressed_basename
-      #   => 'my_file.txt'
-      #
-      # @return [String] the decompressed basename
-      def decompressed_basename
-        basename[0..-(extname.size + 1)]
-      end
-      
-      # The path of this resource after it is decompressed
-      #
-      #   IMW::Resource.new('/path/to/my_file.txt.bz2').decompressed_basename
-      #   => '/path/to/my_file.txt'
-      #
-      # @return [String] the decompressed path
-      def decompressed_path
-        File.join(dirname, decompressed_basename)
-      end
-      
-      # Decompress this file in its present directory overwriting any
-      # existing files and without saving the original compressed
-      # file.
-      #
-      # @return [IMW::Resource] the decompressed resource
-      def decompress!
-        should_exist!("Cannot decompress.")
-        program = compression_settings[:decompression_program] || compression_settings[:program]
-        FileUtils.cd(dirname) { IMW.system(program, compression_settings[:decompress], path) }
-        IMW.open(decompressed_path)
-      end
-
-      # Decompress this file in its present directory, overwriting any
-      # existing files while keeping the original compressed file.
-      #
-      # FIXME The implementation is a little stupid as the file is
-      # needlessly copied.
-      #
-      # @return [IMW::Resource] the decompressed resource
-      def decompress
-        should_exist!("Cannot decompress.")
-        begin
-          copy = cp(path + '.imw_copy')
-          regular_file = decompress!
-          copy.mv(path)
-          regular_file
-        ensure
-          copy.mv(path) if copy && copy.exist?
-        end
-      end
-    end
-  end
-end

data/lib/imw/compressed_files/bz2.rb

@@ -1,16 +0,0 @@
-module IMW
-  module CompressedFiles
-    module Bz2
-
-      include IMW::CompressedFiles::Base
-      
-      def compression_settings
-        @compression_settings ||= {
-          :decompression_program    => :bzip2,
-          :decompress               => '-fd'
-        }
-      end
-      
-    end
-  end
-end

data/lib/imw/compressed_files/compressible.rb

@@ -1,75 +0,0 @@
-module IMW
-
-  # Default settings used when compressing files.  <tt>:program</tt>
-  # defines the name of the command-line program to use,
-  # <tt>:compress</tt> gives the flags to use when compressing, and
-  # <tt>:extension</tt> gives the extension (_without_ the `.') added
-  # by the program after compressing.
-  COMPRESSION_SETTINGS = {
-    :program   => 'bzip2',
-    :compress  => '',
-    :extension => 'bz2'
-  } unless defined?(COMPRESSION_SETTINGS)
-
-  module CompressedFiles
-    
-    # Defines methods for compressing a file.  The default compression
-    # program is defined in IMW::COMPRESSION_SETTINGS though a
-    # particular resource can change the values in its
-    # +compression_settings+ hash.
-    module Compressible
-
-      # Compression settings.
-      attr_accessor :compression_settings
-
-      # Is this file compressible?
-      #
-      # @return [true]
-      def is_compressible?
-        true
-      end
-      
-      # Defines the compression settings used for this
-      # resource. <tt>:program</tt> defines the name of the
-      # command-line program to use, <tt>:compress</tt> gives the
-      # flags to use when compressing, and <tt>:extension</tt> gives
-      # the extension (_without_ the `.') added by the program after
-      # compressing.
-      #
-      # @return [Hash]
-      def compression_settings
-        @compression_settings ||= COMPRESSION_SETTINGS
-      end
-
-      # Compress this resource in place, overwriting it.
-      #
-      # This resource's +compression_settings+ method is used to
-      # determine the method of compression.
-      #
-      # @return [IMW::Resource] the compressed file
-      def compress!
-        should_exist!("Cannot compress.")
-        IMW.system(*[compression_settings[:program], compression_settings[:compress], path])
-        IMW.open(File.join(dirname,basename + "." + compression_settings[:extension]))
-      end
-
-      # Compress this resource without overwriting it.
-      #
-      # FIXME The implementation is a little stupid as the file is
-      # needlessly copied.
-      #
-      # @return [IMW::Resource] the compressed file
-      def compress options={}
-        should_exist!("Cannot compress.")
-        begin
-          copy = cp(path + '.imw_copy')
-          compressed_file = compress!
-          copy.mv(path)
-          compressed_file
-        ensure
-          copy.mv(path) if copy.exist?
-        end
-      end
-    end
-  end
-end

data/lib/imw/compressed_files/gz.rb

@@ -1,16 +0,0 @@
-module IMW
-  module CompressedFiles
-    module Gz
-
-      include IMW::CompressedFiles::Base
-      
-      def compression_settings
-        @compression_settings ||= {
-          :decompression_program  => :gunzip,
-          :decompress             => '-fd'
-        }
-      end
-      
-    end
-  end
-end

data/lib/imw/dataset.rb

@@ -1,125 +0,0 @@
-require 'imw/dataset/workflow'
-require 'imw/dataset/paths'
-
-module IMW
-
-  # The IMW::Dataset represents a common object in which paths, data
-  # resources, and various tasks can be intermingled to define a
-  # complex transformation of data.
-  #
-  # == Organizing Paths
-  #
-  # IMW encourages you to work within the following directory
-  # structure for a dataset +my_dataset+:
-  #
-  #   my_dataset/
-  #   |-- my_dataset.rb
-  #   |-- ripd
-  #   |   `-- ...
-  #   |-- rawd
-  #   |   `-- ...
-  #   |-- fixd
-  #   |   `-- ...
-  #   `-- pkgd
-  #       `-- ...
-  #
-  # Just like IMW itself, a dataset can manage a collection of paths.
-  # If <tt>my_dataset.rb</tt> defines a dataset:
-  #
-  #   # my_dataset/my_dataset.rb
-  #   dataset = IMW::Dataset.new(:my_dataset)
-  #
-  # then the following paths will be defined:
-  #
-  #   dataset.path_to(:root)   #=> my_dataset
-  #   dataset.path_to(:script) #=> my_dataset/my_dataset.rb
-  #   dataset.path_to(:ripd)   #=> my_dataset/ripd
-  #   dataset.path_to(:rawd)   #=> my_dataset/rawd
-  #   dataset.path_to(:fixd)   #=> my_dataset/fixd
-  #   dataset.path_to(:pkgd)   #=> my_dataset/pkgd
-  #
-  # Just like IMW itself, the +dataset+ supports adding path
-  # references
-  #
-  #   dataset.add_path(:raw_data, :ripd, 'raw_data.xml')
-  #   dataset.path_to(:raw_data) #=> my_dataset/ripd/raw_data.xml
-  #
-  # as well as removed (via <tt>dataset.remove_path</tt>)).
-  #
-  # A subclass of IMW::Dataset can customize these paths be overriding
-  # IMW::Dataset#set_default_paths as well as define new ones by
-  # overriding IMW::Dataset#set_paths.
-  #
-  # Setting paths can be skipped altogether by passing the
-  # <tt>:skip_paths</tt> option when instantiating a dataset:
-  #
-  #   dataset = IMW::Dataset.new :my_dataset, :skip_paths => true
-  #
-  # == Utilizing Tasks
-  #
-  # An IMW::Dataset utilizes Rake to manage tasks needed to transform
-  # data.  See IMW::Workflow for a description of the pre-defined
-  # tasks (+rip+, +parse+, +fix+, +package+).
-  #
-  # New tasks can be defined
-  #
-  #   dataset.task :get_authorization do
-  #     # ... get an authorization token
-  #   end
-  #
-  # and hooked into the default tasks in the usual Rake manner
-  #
-  #   dataset.task :rip => [:get_authorization]
-  #
-  # A dataset also has methods for the workflow step tasks to make
-  # this easier
-  #
-  #   dataset.rip [:get_authorized]
-  #
-  # Tasks for a dataset can be accessed and invoked as follows
-  #
-  #   dataset[:rip].invoke
-  #
-  # as well as by using the command line +imw+ tool.
-  #
-  # Defining tasks can be skipped altogether by passing the
-  # <tt>:skip_workflow</tt> option when instantiating a dataset
-  #
-  #   dataset = IMW::Dataset.new :my_dataset, :skip_workflow => true
-  #
-  # == Working with Repositories
-  #
-  # A dataset can be added to a repository by passing the
-  # <tt>:repository</tt> option
-  #
-  #   repo    = IMW::Repository.new
-  #   dataset = IMW::Dataset.new :my_dataset, :repository => repo
-  class Dataset
-
-    # The handle this dataset goes by.  Used for identifying it within
-    # a repository.
-    attr_accessor :handle
-
-    # Options for this dataset.
-    attr_accessor :options
-
-    def initialize handle, options = {}
-      @options = options
-      @handle  = handle
-      set_default_paths   unless options[:skip_paths]
-      set_paths           unless options[:skip_paths]
-      initialize_workflow unless options[:skip_workflow]
-      if options[:repository]
-        options[:repository][handle] = self
-      end
-    end
-
-    # Provides this dataset with a workflow of tasks managed by Rake.
-    include IMW::Workflow
-
-    # Provides this dataset with DSL like methods to construct a
-    # schema in an IMW file.
-    # include IMW::Metadata::DSL
-
-  end
-end

data/lib/imw/dataset/paths.rb

@@ -1,29 +0,0 @@
-module IMW
-  class Dataset
-    include IMW::Paths
-
-    protected
-    # Sets paths to the workflow directories for this dataset (+ripd+,
-    # +rawd+, +fixd+, +pkgd+) as well as the following paths:
-    #
-    # script::
-    #   The path to the file the dataset was initialized in.
-    #
-    # root::
-    #   The parent directory of the file the dataset was initialized
-    #   in or the value of the <tt>:root</tt> key in
-    #   IMW::Dataset#options
-    #
-    def set_default_paths
-      add_path :script, File.expand_path(eval('__FILE__'))
-      add_path :root,   options[:root] || File.dirname(path_to(:script))
-      workflow_dirs.each do |dir|
-        add_path dir, :root, dir.to_s
-      end
-    end
-
-    # Overwrite this method to set additional paths for the dataset.
-    def set_paths
-    end
-  end
-end

data/lib/imw/dataset/workflow.rb

@@ -1,195 +0,0 @@
-require 'ostruct'
-require 'rake'
-
-module IMW
-
-  # An IMW version of Rake::Task
-  Task             = Class.new(Rake::Task)
-
-  # An IMW subclass of Rake:FileTask
-  FileTask         = Class.new(Rake::FileTask)
-
-  # An IMW subclass of Rake::FileCreationTask
-  FileCreationTask = Class.new(Rake::FileCreationTask)
-
-  # IMW encourages you to view a data transformation as a series of
-  # interdependent steps.
-  #
-  # By default, IMW defines four main steps in such a transformation:
-  # +rip+, +parse+, +fix+, and +package+.
-  #
-  # Each step is associated with a directory on disk in which it keeps
-  # its files: +ripd+, +prsd+, +fixd+, and +pkgd+.
-  #
-  # The steps are:
-  #
-  # rip::
-  #   Obtain data via HTTP, FTP, SCP, RSYNC, database query, &c and
-  #   store the results in +ripd+.
-  #
-  # parse::
-  #   Parse data into a structured form using a library (JSON, YAML,
-  #   &c.) or using your own parser (XML, flat files, &c.) and store
-  #   the results in +prsd+
-  #
-  # fix::
-  #   Combine, filter, reconcile, and transform already structured
-  #   data into a desired form and store the results in +fixd+.
-  #
-  # package::
-  #   Archive, compress, and deliver data in its final form to some
-  #   location (HTTP, FTP, SCP, RSYNC, S3, EBS, &c.), optionally
-  #   storing the ouptut in +pkgd+.
-  #
-  # Each step depends upon the one before it.  The steps are blank by
-  # default so there's no need to write code for steps you don't need
-  # to use.  You can also define your own steps (using +task+ just
-  # like in Rake) and hook them into these pre-defined steps (or
-  # not...).
-  #
-  # A dataset also has an <tt>:initialize</tt> task (which by default
-  # just creates the directories for these steps) which you can use to
-  # hook in your own initialization tasks by making it depend on them.
-  #
-  # A subclass of IMW::Dataset can customize how tasks are defined by
-  # overriding +define_workflow_tasks+, among other methods, and
-  # introduce new tasks by overriding +define_tasks+.
-  module Workflow
-
-    include Rake::TaskManager
-    # Default options passed to <tt>Rake</tt>.  Any class including
-    # the <tt>Rake::TaskManager</tt> module must define a constant by
-    # this name.
-    DEFAULT_OPTIONS = {
-      :dry_run => false,
-      :trace   => false,
-      :verbose => false
-    }
-    
-    # Return a new (or existing) <tt>IMW::Task</tt> with the given
-    # +name+.  Dependencies can be declared and a block passed in just
-    # as in Rake.
-    #
-    # @param [Hash, Symbol, String] deps the name of the task (if a
-    # Symbol or String) or the name of the task mapped to an Array of
-    # dependencies (if a Hash)
-    #
-    # @return [IMW::Task] the task
-    def task deps, &block
-      self.define_task IMW::Task, deps, &block
-    end
-
-    # Return a new (or existing) <tt>IMW::FileTask</tt> with the given
-    # +path+.  Dependencies can be declared and a block passed in just
-    # as in Rake.
-    #
-    # @param [String, IMW::Resource] path the path to the file
-    # @return [IMW::FileTask] the task
-    def file path, &block
-      path = path.respond_to?(:path) ? path.path : path
-      self.define_task IMW::FileTask, path, &block
-    end
-
-    # Return a new (or existing) <tt>IMW::FileCreationTask</tt> with the given
-    # +path+.  Dependencies can be declared and a block passed in just
-    # as in Rake.
-    #
-    # @param [String, IMW::Resource] path the path to the file
-    # @return [IMW::FileCreationTask] the task
-    def file_create path, &block
-      path = path.respond_to?(:path) ? path.path : path
-      self.define_task IMW::FileCreationTask, path, &block
-    end
-
-    # Override this method to define default tasks for a subclass of
-    # IMW::Dataset.
-    def define_tasks
-    end
-
-    # The standard IMW workflow steps.
-    #
-    # @return [Array] the workflow step names
-    def workflow_steps
-      [:rip,  :parse, :fix, :package]
-    end
-
-    # The steps of the IMW workflow each correspond to a directory in
-    # which it is customary that they deposit their files <em>once
-    # they are finished processing</em> (so ripped files wind up in
-    # the +ripd+ directory, packaged files in the +pkgd+ directory,
-    # and so on).
-    #
-    # @return [Array] the workflow directory names
-    def workflow_dirs
-      [:ripd, :rawd,  :fixd, :pkgd]
-    end
-
-    protected
-    
-    # Convenience method for defining tasks for this workflow.
-    #
-    # @param [Hash, Symbol, String] deps the name of the task (if a
-    # Symbol or String) or the name of the task mapped to an Array of
-    # dependencies (if a Hash)
-    # @param [String] comment the comment to associate to the task
-    # @return [IMW::Task] the task
-    def define_workflow_task deps, comment, &block
-      @last_description = comment
-      define_task(IMW::Task, deps, &block)
-    end
-
-    # Create all the instance variables required by Rake::TaskManager
-    # and define default tasks for this dataset.
-    def initialize_workflow
-      @tasks = Hash.new
-      @rules = Array.new
-      @scope = Array.new
-      @last_description = nil
-      @options = OpenStruct.new(DEFAULT_OPTIONS)
-      define_initialize_task
-      define_workflow_tasks
-      define_clean_task
-      define_tasks
-    end
-
-    # Defines the <tt>:initialize</tt> task.  The only other task
-    # hooked into <tt>:initialize</tt> is the
-    # <tt>:create_workflow_dirs</tt> task which creates the workflow
-    # directories for this dataset.
-    def define_initialize_task
-      define_workflow_task({:create_directories => []}, "Creates workflow directories for this dataset.") do
-        workflow_dirs.each do |dir|
-          FileUtils.mkdir_p(path_to(dir)) unless File.exist?(path_to(dir))
-        end
-      end
-      define_workflow_task({ :initialize => [:create_directories] }, "Initialize this dataset.")
-    end
-
-    # Creates a task <tt>:clean</tt> which removes dataset's
-    # workflow directories.
-    def define_clean_task
-      define_workflow_task :clean, "Remove the workflow directories for this dataset." do
-        workflow_dirs.each do |dir|
-          FileUtils.rm_rf(path_to(dir)) if File.exist?(path_to(dir))
-        end
-      end
-    end
-
-    # Creates the task dependency chain <tt>:package => :fix =>
-    # :parse => :rip => :initialize</tt> of the
-    # IMW::Workflow.
-    def define_workflow_tasks
-      define_workflow_task({:rip     => [:create_directories]}, "Obtain data from some source."           )
-      define_workflow_task({:parse   => [:rip]},                "Parse data into a structured form."      )
-      define_workflow_task({:fix     => [:parse]},              "Munge parsed data into desired form."    )
-      define_workflow_task({:package => [:fix]},                "Package dataset in final form."          )
-    end
-
-
-    def rip(deps=nil, &block);     self[:rip].enhance(deps, &block);     end
-    def parse(deps=nil, &block);   self[:parse].enhance(deps, &block);   end
-    def fix(deps=nil, &block);     self[:fix].enhance(deps, &block);     end
-    def package(deps=nil, &block); self[:package].enhance(deps, &block); end
-
-  end
-end