imw 0.1.0

data/README.rdoc

@@ -0,0 +1,101 @@
+
+= Overview
+
+The Infinite Monkeywrench (IMW) is a Ruby frameworks to simplify the
+tasks of acquiring, extracting, transforming, loading, and packaging
+data.  It has the following goals:
+
+* Minimize programmer time even at the expense of increasing run
+  time.
+
+* Take data through a full transformation from raw source to packaged
+  purity in as few lines of code as possible.
+
+* Treat data records as objects as much as possible.
+
+* Use instead of repeat better code that already exists in other
+  libraries (FasterCSV, I'm talkin' to you).
+
+* Make what's common easy without making what's uncommon impossible.
+
+* Work with messy data as well as clean data.
+
+* Let you incorporate your own tools wherever you choose to.
+
+The Infinite Monkeywrench is a powerful tool but it is not always the
+right one to use.  IMW is **not** designed for
+
+* Scraping vast amounts of data (use Wuclan[http://github.com/infochimps/wuclan], Monkeyshines[http://github.com/infochimps/monkeyshines], and Edamame[http://github.com/infochimps/edamame].)
+
+* Really, really big datasets (use Wukong[http://github.com/infochimps/wukong] and Hadoop[http://hadoop.apache.org])
+
+* Data mining
+
+* Data visualization
+
+
+= Setup
+
+IMW is hosted on Gemcutter[http://gemcutter.org] so it's easy to install.
+
+You'll have to set up Gemcutter
+
+  $ sudo gem install gemcutter
+  $ gem tumble
+
+and then install IMW
+
+  $ sudo gem install imw
+
+= Using IMW
+
+The central goal of IMW is to make workflow involved in processing a
+dataset from a raw source to a finished product as simple as possible.
+
+So consider that there exist two datasets that I want to combine.  The
+first details the historical price of bananas over the past century
+and the second 
+
+== Working with paths and files
+
+  require 'rubygems'
+  require 'imw'
+
+IMW holds a registry of paths that you can define on the fly or store
+in a configuration file.
+
+  IMW.add_path :dropbox, "/var/www/public/dropbox"
+  IMW.add_path :raw,     "/mnt/data/raw"
+  IMW.add_path :
+
+This makes it easeir
+
+  IMW.path_to :raw, "one/particular/dataset"
+  #=> "/mnt/data/raw/one/particular/dataset"
+
+IMW makes it easy to manipulate compressed files and archives.
+
+
+  # Move a collection of files from a public dropbox to a processing directory
+
+  raw
+  
+  Dir["/public/*"].each do |path|
+    file = IMW.open(path)
+    case
+    when file.compressed?
+      file.decompress.mv_to_dir "/raw"
+    when file.archive?
+      FileUtils.cd("/raw") do
+        file.extract
+      end
+    else
+      file.mv_to_dir("/raw")
+    end
+  end
+  
+  
+	
+    
+  
+

data/Rakefile

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  # http://github.com/technicalpickles/jeweler
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "imw"
+    gem.summary = "The Infinite Monkeywrench (IMW) makes acquiring, extracting, transforming, loading, and packaging data easy."
+    gem.description = "The Infinite Monkeywrench (IMW) is a Ruby frameworks to simplify the tasks of acquiring, extracting, transforming, loading, and packaging data. It minimizes programmer time by encapsulating common data workflows and patterns and creating interfaces to many other useful Ruby libraries."
+    gem.email = "coders@infochimps.org"
+    gem.homepage = "http://github.com/infochimps/imw"
+    gem.authors = ["Dhruv Bansal", "Philip (flip) Kromer"]
+    
+    gem.files.exclude "old/**/*"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available.  Install it with: sudo gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/etc/imwrc.rb

@@ -0,0 +1,76 @@
+#-*- mode: ruby -*-
+#
+# h2. etc/imwrc -- default site-wide imw configuration file
+#
+# == About
+#
+# This file contains the site-wide configuration settings for this
+# installation of the Infinite Monkeywrench.  Settings here override
+# the defaults in <tt>lib/imw/utils/config.rb</tt> (see the
+# documentation for that file for more detail on the variables that
+# can be configured here) but will in turn be overwritten by settings
+# in the <tt>~/.imwrc</tt> file in each user's directory (though the
+# location of this file can be customized).
+#
+# At the present moment, all settings are stored as plain Ruby files
+# (though they may lack the <tt>.rb</tt> extension).  As the IMW
+# develops, these will be replaced by YAML files which will be parsed
+# by <tt>lib/imw/utils/config.rb</tt>.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+
+module IMW
+  PATHS = {
+    :home         => ENV['HOME'],
+    :data_root    => "/var/lib/imw",
+    :log_root     => "/var/log/imw",
+    :scripts_root => "/usr/share/imw",
+    :tmp_root     => "/tmp/imw",
+
+    # the imw library
+    :imw_root  => File.expand_path(File.dirname(__FILE__) + "/.."),
+    :imw_bin   => [:imw_root, 'bin'],
+    :imw_etc   => [:imw_root, 'etc'],
+    :imw_lib   => [:imw_root, 'lib'],
+
+    # workflow
+    :ripd_root  => [:data_root, 'ripd'],
+    :peeld_root => [:data_root, 'peeld'],
+    :mungd_root => [:data_root, 'mungd'],
+    :temp_root  => [:data_root, 'temp'],
+    :fixd_root  => [:data_root, 'fixd'],
+    :pkgd_root  => [:data_root, 'pkgd']
+  }
+
+  # Default time format.
+  STRFTIME_FORMAT = "%Y%m%d-%H%M%S" unless defined? STRFTIME_FORMAT
+
+  # Paths to external programs used by IMW.
+  EXTERNAL_PROGRAMS = {
+    :tar => "tar",
+    :rar => "rar",
+    :zip => "zip",
+    :unzip => "unzip",
+    :gzip => "gzip",
+    :bzip2 => "bzip2",
+    :wget => "wget"
+  } unless defined? ::IMW::EXTERNAL_PROGRAMS
+
+  module Files
+    # Regular expressions which match pathnames to the name of the
+    # appropriate IMW::Files class.
+    #
+    # File class names should be stripped of the leading
+    # <tt>IMW::Files</tt> prefix, i.e. - the file object
+    # <tt>IMW::Files::Bz2</tt> should be referenced by the string
+    # <tt>"Bz2"</tt>.
+    FILE_REGEXPS = [] unless defined? ::IMW::Files::FILE_REGEXPS
+  end
+
+end
+
+

data/lib/imw.rb

@@ -0,0 +1,42 @@
+#
+# h2. lib/imw.rb -- main imw file
+#
+# == About
+#
+# This file is the entry-point to the IMW library.  It loads a minimal
+# setup.  Optional components can be loaded by calling the function
+# <tt>IMW.imw_components</tt>.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+# puts "#{File.basename(__FILE__)}: Behold, the weighty, the munificent, the Infinite Monkeywrench! Approach it with care: it has overwhelmed mightier monkeys than ye."
+
+require 'rubygems'
+require 'YAML' unless defined?('YAML') # some stupid collision with datamapper makes it double include
+require 'imw/boot'
+require 'imw/utils'
+require 'imw/dataset'
+require 'imw/files'
+require 'imw/parsers'
+require 'imw/packagers'
+
+# The Infinite Monkeywrench (IMW) is a Ruby library for obtaining,
+# parsing, transforming, reconciling, and packaging datasets.
+#
+# Data is obtained via FIXME
+#
+# Data is loaded into IMW using <tt>IMW.open</tt> which provides a
+# uniform interface across a variety of data formats.  The objects
+# returned will each have +load+ method which will return data in the
+# best form for further processing.  If the data is a YAML file, then
+# Ruby's +YAML+ library will be used to return primitive Ruby objects,
+# if it is a CSV, then the +FasterCSV+ library will be used, &c.
+#
+# The main interface to handling data is the <tt>IMW::Dataset</tt>
+# class.  It has methods for summarizing, transforming, and dumping
+# data to a variety of formats.
+module IMW
+end

data/lib/imw/boot.rb

@@ -0,0 +1,58 @@
+#
+# h2. lib/imw/boot.rb -- startup functions
+#
+# == About
+#
+# This file contains code necessary to boot the Infinite Monkeywrench
+# at a particular site.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+# puts "#{File.basename(__FILE__)}: You heft up your Infinite Monkeywrench for the first time and marvel at how something so powerful could be made so wondrous light!"
+
+module IMW
+  module Config
+
+    # Root of the IMW source base.
+    def self.imw_root
+      File.expand_path File.join(File.dirname(__FILE__), '../..')
+    end
+
+    #
+    # User configuration file
+    #
+    # By default, the file ~/.imwrc (.imwrc, in your home directory -- note no .rb extension)
+    # is sourced at top level.  If the $IMWRC environment variable is set,
+    # that file will be sourced instead.
+    #
+    # Any code within this file will override settings in IMW_ROOT/etc/imwrc.rb
+    #
+    USER_CONFIG_FILE = File.join(ENV['HOME'] || '', '.imwrc')
+    # Environment variable to override user configuration file location.
+    ENV_CONFIG_FILE = "IMWRC"
+    def self.user_config_file # :nodoc:
+      File.expand_path(ENV[ENV_CONFIG_FILE] || USER_CONFIG_FILE)
+    end
+
+    # System-level config file
+    SITE_CONFIG_FILE = "etc/imwrc.rb"
+    def self.site_config_file # :nodoc:
+      File.join(imw_root, SITE_CONFIG_FILE)
+    end
+
+    # Source the config files
+    def self.load_config
+      require site_config_file
+      load    user_config_file   if File.exist? user_config_file
+    end
+  end
+end
+
+#
+# Load the config files
+#
+IMW::Config.load_config
+

data/lib/imw/dataset.rb

@@ -0,0 +1,233 @@
+#
+# h2. lib/imw/dataset.rb -- imw dataset
+#
+# == About
+#
+# Defines basic properties of the <tt>IMW::Dataset</tt>
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+# puts "#{File.basename(__FILE__)}: You use your Monkeywrench to rake deep and straight furrows in the earth for your orchard." # at bottom
+
+require 'rake'
+require 'ostruct'
+
+require 'imw/utils'
+require 'imw/dataset/workflow'
+require 'imw/dataset/loaddump'
+require 'imw/dataset/stats'
+
+module IMW
+
+  # The basic unit in IMW is the dataset.  Each dataset has a handle
+  # which is meant to be unique (at least in the context of a
+  # particular pool of datasets, see <tt>IMW::Pool</tt>).  A dataset
+  # can also have a taxonomic classification or _taxon_
+  #
+  #   dataset = IMW::Dataset.new :recent_history_of_banana_prices,
+  #                              :taxon => [:economics,:alarming_trends]
+  #
+  # but it isn't required like the handle.
+  #
+  # Processing a dataset commonly occurs in four course steps.  IMW
+  # defines a task[http://rake.rubyforge.org] for each of these steps
+  # and keeps files involved in different steps in different
+  # directories.
+  #
+  # rip::
+  #   Managed by the <tt>:rip</tt> task, data is collected from a
+  #   source (+http+, +ftp+, database, &c.) and deposited in a
+  #   subdirectory of the <tt>:ripd</tt> directory named for the URI
+  #   of the source.
+  #
+  #     dataset.task :rip do
+  #       IMW::Rip.from_web 'http://econ.chimpu.edu/datasets/produce_prices.tar.bz2'
+  #         #=> [ripd]/http/econ_chimpu_edu/datasets/produce_prices.tar.bz2
+  #         
+  #       IMW::Rip.from_database :named  => "weather_records",
+  #                              :at     => "public.astro.chimpu.edu",
+  #                              :select => "* FROM hurricane_frequency"
+  #         #=> [ripd]/sql/_edu/chimpu_astro_public/weather_records/select_from_hurricane_frequency-2009-02-16--15:30:26.tsv
+  #     end
+  #
+  #   Where <tt>[ripd]</tt> would be replaced by the IMW
+  #   <tt>:ripd</tt> directory.  The default <tt>:rip</tt> task is
+  #   empty so If there's no need to rip data (perhaps it's already on
+  #   disk?) then nothing needs to be done here.
+  #   
+  # raw::
+  #   Managed by the <tt>:raw</tt> task, data is uncompressed and
+  #   extracted (if necessary) and stored in a subdirectory of the
+  #   <tt>:data</tt> directory named by the taxon and handle of this
+  #   dataset.
+  #
+  #     dataset.task :raw do
+  #       IMW::Raw.uncompress_and_extract File.join(dataset.path_to(:ripd),'http/_edu/chimpu_econ/datasets'),
+  #                                       Dir[File.join(dataset.path_to(:ripd),'sql/_edu/chimpu_astro_public/**/*.tsv')].first
+  #       #=> [data]/economics/alarming_trends/recent_history_of_banana_prices/rawd/001.xml
+  #           [data]/economics/alarming_trends/recent_history_of_banana_prices/rawd/002.xml
+  #           [data]/economics/alarming_trends/recent_history_of_banana_prices/rawd/003.xml
+  #           ...
+  #           [data]/economics/alarming_trends/recent_history_of_banana_prices/rawd/select_from_hurricane_frequency-2009-02-16--15:30:26.tsv
+  #     end
+  #
+  #   Where <tt>[data]</tt> would be replaced by the IMW
+  #   <tt>:data</tt> directory.
+  #
+  #   If this dataset didn't have a taxon
+  #   (economics/alarming_trends) its files would be stored in a
+  #   directory +recent_history_of_banana_prices+ just below the
+  #   <tt>:data</tt> directory.
+  #
+  # fix::
+  #   Managed by the <tt>:fix</tt> task, transformations on the data
+  #   are performed.  IMW's method is to read data from a source
+  #   format (XML, YAML, CSV, &c.) into Ruby objects with hash
+  #   semantics.  These objects might be based upon structs,
+  #   ActiveRecord, DataMapper::Resource, FasterCSV...anything which
+  #   can be accessed as <tt>thing.property</tt> (FIXME 'and' or 'or'
+  #   ) <tt>thing[:property]</tt>: the Infinite Monkeywrench fits
+  #   neatly into your toobox.
+  #
+  #
+  #     # Open an output file in XML for writing
+  #     output = IMW.open! File.join(dataset.path_to(:fixd), 'date_bananas_hurricanes.csv')
+  #       #=> FasterCSV at [fixd]/economics/alarming_trends/recent_history_of_banana_prices/fixd/data_bananas_hurricanes.csv
+  #
+  #     # A place to store the combined data
+  #     correlations = []
+  #
+  #     dataset.task :fix do
+  #
+  #       # Return the contents of the weather data which has rows like
+  #       # 
+  #       #   1    2008-09-01    4
+  #       #   2    2008-09-08    3
+  #       #   3    2008-08-15    3
+  #       #   ...  
+  #       # 
+  #       weather_data = IMW.open(Dir[File.join(dataset.path_to(:rawd), '*.tsv')].first,
+  #                               :headers => ["ID","DATE","NUM_HURRICANES"]).entries
+  #         #=> [#<FasterCSV::Row "ID":nil "DATE":Mon Sep 08 04:15:47 -0600 2008,"NUM_HURRICANES":4>, ... ]
+  #
+  #
+  #       # Return the matching data from the produce prices XML file which looks like
+  #       # 
+  #       #  <prices>
+  #       #    <price type="apple">
+  #       #      <date>2008/09/01</date>
+  #       #      <amount>0.15</amount>
+  #       #    </price>
+  #       #    <price type="banana">
+  #       #      <date>2008/09/01</date>
+  #       #      <amount>0.20</amount>
+  #       #    </price>
+  #       #    ...
+  #       #  </prices>
+  #       parser = IMW::XMLParser.new :records => [ 'prices/price[@type="banana"]',
+  #                                                 { :week  => 'date',
+  #                                                   :price => 'amount' }]
+  #
+  #       # Loop through the XML produce prices, mixing in the hurricane data,
+  #       # and outputting new rows.
+  #       Dir["#{dataset.path_to :rawd}*.xml"] each do |file|
+  #         IMW.open file do |xml| #=> Hpricot::Doc
+  #           parser.parse(xml).each do |record|
+  #             num_hurricanes = weather_data.(lambda { nil }) {|id,week,num_hurricanes| week == record.week}
+  #             output << [week,record[:price],num_hurricanes]
+  #           end
+  #         end
+  #       end
+  #     end
+  #
+  # package::
+  #   Data is packaged and compressed (if necessary) into a delivery
+  #   format and deposited into the <tt>:pkgd</tt> directory.
+  #
+  #   dataset.task :pkg do
+  #     IMW.open(File.join(dataset.path_to(:fixd), 'date_bananas_hurricanes.csv')).compress!
+  #       #=> [data]/economics/alarming_trends/recent_history_of_banana_prices/pkgd/date_bananas_hurricanes.csv.bz2
+  #   end
+  #
+  # In the above, <tt>dataset.task</tt> behaves like
+  # <tt>Rake.task</tt>, merely defining a task and its dependencies
+  # without executing it via
+  #
+  #   dataset.task(:pkg).invoke
+  #
+  # Since the <tt>:rip</tt>, <tt>:raw</tt>, <tt>:fix</tt>, and
+  # <tt>:pkg</tt> tasks depend upon each other, invoking <tt>:pkg</tt>
+  # will first cause <tt>:rip</tt> to run.
+  #
+  # By default, the tasks associated with a dataset are blank.  All of
+  # IMW's functionality is available without defining tasks.  Tasks
+  # simply provide a convenient scaffold for building a data
+  # transformation upon.
+  #
+  # Similarly, there is no requirement to use the directory structure
+  # outlined above.  IMW's methods accept plain filenames and do the
+  # Right Thing where possible.  The combination of tasks with
+  # matching directory structure is a suggested but not mandatory
+  # framework in which to program.
+  class Dataset
+
+    # The <tt>Rake::TaskManager</tt> module allows the
+    # <tt>IMW::Dataset</tt> class to leverage the functionality of the
+    # Rake[http://rake.rubyforge.org/] library to manage tasks
+    # associated with the processing of this dataset.
+    include Rake::TaskManager
+
+    # The <tt>IMW::Workflow</tt> module contains pre-defined tasks for
+    # dataset processing.
+    include IMW::Workflow
+
+    attr_reader   :handle, :taxon, :options
+    attr_accessor :data
+
+    # The default taxon assigned to a dataset.
+    DEFAULT_TAXON = nil
+
+    # Default options passed to <tt>Rake</tt>.  Any class including
+    # the <tt>Rake::TaskManager</tt> module must define a constant by
+    # this name.
+    DEFAULT_OPTIONS = {
+      :dry_run => false,
+      :trace   => false,
+      :verbose => false
+    }
+
+    # Create a new dataset.  Arguments include
+    #
+    #   <tt>:taxon</tt> (+DEFAULT_TAXON+):: a string or sequence
+    #   giving the taxonomic classification of the dataset.  See
+    #   <tt>IMW::Dataset.taxon=</tt> for more details on how this
+    #   argument is interpreted.
+    def initialize handle, options = {}
+      options = options.reverse_merge :taxon => DEFAULT_TAXON
+
+      # FIXME is this how the attribute writer functions should be
+      # called?
+      @handle = handle
+      @taxon = options[:taxon]
+
+      # for rake
+      @tasks = Hash.new
+      @rules = Array.new
+      @scope = Array.new
+      @last_description = nil
+      @options = OpenStruct.new(DEFAULT_OPTIONS)
+      create_default_tasks
+
+      # sets an empty @paths hash; see utils/paths.rb
+      set_paths
+    end
+
+    def handle= thing
+      @handle = thing.is_a?(String) ? thing.to_handle : thing
+    end
+
+  end
+end