imw 0.1.0

data/lib/imw/utils.rb

@@ -0,0 +1,24 @@
+#
+# h2. lib/imw/utils.rb -- utility functions
+#
+# == About
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+
+require 'rubygems'
+require 'imw/utils/error'
+require 'imw/utils/log'
+require 'imw/utils/config'
+require 'imw/utils/paths'
+require 'imw/utils/misc'
+require 'imw/utils/components'
+require 'imw/utils/extensions/core'
+require 'fileutils'
+require 'pathname'
+
+
+# puts "#{File.basename(__FILE__)}: Early economists thought they would measure the utility of an action in units of `utils'.  Really." # at bottom

data/lib/imw/utils/components.rb

@@ -0,0 +1,61 @@
+#
+# h2. lib/imw/utils/components.rb -- define separate components of IMW
+#
+# == About
+#
+# Defines a hash <tt>IMW::COMPONENTS</tt> which keys component names
+# to the files to be required to implement each component and defines
+# methods to load these files.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+# puts "#{File.basename(__FILE__)}: Something clever" # at bottom
+
+require 'imw/utils/error'
+
+module IMW
+
+  # Defines IMW components and the files required by each.  Components
+  # can be accessed using <tt>IMW.load_components</tt> or
+  # <tt>IMW#imw_components</tt>.
+  COMPONENTS = {
+    :datamapper => ["imw/dataset/datamapper","imw/dataset/datamapper/time_and_user_stamps"],
+    :data_mapper => :datamapper,
+    :html_parser => "imw/parsers/html_parser",
+    :flat_file_parser => "imw/parsers/flat_file_parser",
+    :line_parser => "imw/parsers/line_parser",
+    :infochimps => ["imw/infochimps/infochimps_resource","imw/infochimps/icss"]
+  }
+
+  # Load components of IMW as needed,
+  #
+  #   IMW.load_components :datamapper, :flat_file_parser
+  def self.load_components *args
+    args.each do |component_name|
+      begin
+        require component_name.to_s
+      rescue LoadError
+        component = IMW::COMPONENTS[component_name]
+        raise IMW::Error.new("#{component_name} is an invalid IMW component.  See IMW::COMPONENTS.") unless component        
+        if component.is_a? Array then
+          IMW.load_components *component
+        else
+          IMW.load_components component
+        end
+      end
+    end
+  end
+
+  # Load components of IMW as needed,
+  #
+  #   include IMW
+  #   imw_components :datamapper, :flat_file_parser
+  def imw_components *args
+    IMW.load_components *args
+  end
+
+end
+

data/lib/imw/utils/config.rb

@@ -0,0 +1,46 @@
+#
+# h2. lib/imw/utils/config.rb -- configuration parsing
+#
+# == About
+#
+# IMW looks for configuration settings in the following places, in
+# order of increasing precedence:
+#
+#   1. Settings defined directly in this file.
+#
+#   2. From the <tt>etc/imwrc</tt> file in the IMW root directory.
+#
+#   3. From the <tt>.imwrc</tt> file in the user's home directory (the
+#      filename can be changed; see
+#      <tt>IMW::Config::USER_CONFIG_FILE_BASENAME</tt>).
+#
+#   4. From the file defined by the environment variable +IMWRC+ (the
+#      value can be changed; see
+#      <tt>IMW::Config::USER_CONFIG_FILE_ENV_VARIABLE</tt>
+#
+# Settings not found in one configuration location will be searched
+# for in locations of lesser precedence.
+#
+# *Note:* configuration files are plain Ruby code that will be directly
+# evaluated.
+#
+# Relevant settings include
+#
+# * interfaces with external programs (+tar+, +wget+, &c.)
+# * paths to directories where IMW reads/writes files
+# * correspondences between file extensions and IMW file classes
+#
+# For more detailed information, see the default configuration file,
+# <tt>etc/imwrc</tt>.
+#
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+
+module IMW
+end
+
+# puts "#{File.basename(__FILE__)}: You carefully adjust the settings on your Monkeywrench.  Glob-monsters: beware!!" # at bottom

data/lib/imw/utils/error.rb

@@ -0,0 +1,54 @@
+#
+# h2. lib/imw/utils/error -- errors
+#
+# == About
+#
+# Error objects for IMW.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+
+module IMW
+
+  # A generic error class.
+  class Error < StandardError
+  end
+
+  class TypeError < TypeError
+  end
+
+  class ArgumentError < ArgumentError
+  end
+
+  class NotImplementedError < NotImplementedError
+  end
+
+  class ParseError < Error
+  end
+
+  # An error meant to be used when a system call goes awry.  It will
+  # report exit status and the process id of the offending call.
+  class SystemCallError < IMW::Error
+
+    def initialize(message)
+      @message = message
+    end
+
+    def display
+      "(error code: #{$?.exitstatus}, pid: #{$?.pid}) #{@message}"
+    end
+
+    def to_s
+      "(error code: #{$?.exitstatus}, pid: #{$?.pid}) #{@message}"
+    end
+
+  end
+
+  # A error for improperly specified, inappropriate, or broken paths.
+  class PathError < IMW::Error
+  end
+
+end

data/lib/imw/utils/extensions/array.rb

@@ -0,0 +1,125 @@
+#
+# h2. lib/imw/utils/extensions/array.rb -- array extensions
+#
+# == About
+#
+# Extensions to the +Array+ class.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+require 'active_support/core_ext/array/extract_options'
+class Array #:nodoc:
+  include ActiveSupport::CoreExtensions::Array::ExtractOptions
+end
+
+class Array
+
+  # Return all but the last  element
+  # This will be [] for both an empty array and a length-1 array
+  def most() self[0..-2] end
+
+  # Return all but the first element.
+  # This will be nil for an empty array and [] for a length-1 array
+  def rest() self[1..-1] end
+
+  # 'Un'-zip()s an array.  Returns an array of arrays: the first array has the
+  # first element of each member, the second array has the second element of
+  # each member, and so on.  Returns as many arrays as the first element in self
+  # and inserts a nil where the member array wasn't long enough.
+  #
+  # foo, bar = foo.zip(bar).unzip should leave foo and bar with the same values
+  # if foo and bar have the same length.
+  #
+  # Will fail on a not-array-of-arrays.
+  def unzip()
+    # An array of empty arrays, one for each vertical slot
+    vslices = self[0].map{ Array.new }
+    self.each do |hslice|
+      # push the elements of each array onto its slice.
+      vslices.zip(hslice).map{|vslice,h_el| vslice << h_el }
+    end
+    vslices
+  end
+
+  # Return a random element of this array.
+  def random_element
+    self[rand(self.length) - 1]
+  end
+
+  # convert an assoc (list of [key, val, [...]]'s) to a hash
+  def to_openstruct
+    mapped = {}
+    each{ |key,value| mapped[key] = value.to_openstruct }
+    OpenStruct.new(mapped)
+  end
+
+  # Return the elements of this array in a pretty-printed string,
+  # inserting +final_string+ between the last two items.
+  #
+  #   >> [:one, :two, :three].quote_items_with "or"
+  #   `one', `two', or `three'
+  #
+  def quote_items_with final_string = nil
+    string_items = self.map { |item| "`" + item.to_s + "'" }
+    case string_items.length
+    when 0
+      ""
+    when 1
+      string_items.first
+    when 2
+      if final_string then
+        string_items.join(" #{final_string} ")
+      else
+        string_items.join(', ')
+      end
+    else
+      string = string_items[0,string_items.length - 1].join ', '
+      if final_string then
+        string += ', ' + final_string + ' ' + string_items.last
+      else
+        string += ', ' + string_items.last
+      end
+      string
+    end
+  end
+
+  def in_groups_of(number, fill_with = nil, &block)
+    require 'enumerator'
+    collection = dup
+    collection << fill_with until collection.size.modulo(number).zero?
+    collection.each_slice(number, &block)
+  end
+
+  # Returns a single hash containing the merge of all hashes in this
+  # array.  This is useful when dealing with badly written YAML files.
+  # Only merges hashes at depth zero, i.e. - this isn't recursive.
+  def merge_hashes
+    merged_hash = {}
+    self.each do |element|
+      merged_hash.merge!(element) if element.is_a?(Hash)
+    end
+    merged_hash
+  end
+
+  # Recurses through the elements of this Array collecting all String
+  # or Symbol "terminal" nodes.
+  def terminals &block
+    terminals = []
+    each do |element|
+      if element.respond_to? :terminals then
+        terminals += element.terminals
+      else
+        terminals << element
+      end
+    end
+    terminals.map! {|terminal| yield terminal } if block
+    terminals
+  end
+  
+
+end
+
+# puts "#{File.basename(__FILE__)}: I have a loooong list of complaints.  Firstly, ..." # at bottom

data/lib/imw/utils/extensions/class/attribute_accessors.rb

@@ -0,0 +1,8 @@
+# :nodoc:
+# for when cattr_accessor is all you need
+#
+require 'active_support/core_ext/array/extract_options'
+class Array #:nodoc:
+  include ActiveSupport::CoreExtensions::Array::ExtractOptions
+end
+require 'active_support/core_ext/class/attribute_accessors'

data/lib/imw/utils/extensions/core.rb

@@ -0,0 +1,43 @@
+#
+# h2. lib/imw/utils/extensions/core.rb -- extensions to the Ruby core
+#
+# == About
+#
+# Some useful extensions to basic Ruby classes.  This file is required
+# by <tt>imw/utils</tt> so any files required here are automatically
+# required when loading IMW.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+# puts "#{File.basename(__FILE__)}: Your monkeywrench does a complicated series of core-burning exercises and emerges with ripped, powerful-looking abs."
+
+require 'imw/utils/extensions/string'
+require 'imw/utils/extensions/array'
+require 'imw/utils/extensions/hash'
+require 'imw/utils/extensions/dir'
+require 'imw/utils/extensions/struct'
+require 'imw/utils/extensions/symbol'
+require 'imw/utils/extensions/file_core'
+require 'active_support/core_ext/module/aliasing'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/object/misc'
+#require 'active_support/core_ext/blank.rb'
+require 'imw/utils/extensions/class/attribute_accessors'
+# require 'ostruct'
+require 'set'
+
+module IMW
+  # A replacement for the standard system call which raises an
+  # IMW::SystemCallError if the command fails as well as printing the
+  # command appended to the end of <tt>error_message</tt>.
+  def self.system *commands
+    command = commands.flatten.join ' '
+    Kernel.system(command)
+    raise IMW::SystemCallError.new(command) unless $?.success?
+  end
+end
+
+

data/lib/imw/utils/extensions/dir.rb

@@ -0,0 +1,24 @@
+#
+# h2. lib/imw/utils/extensions/dir.rb -- directory extensions
+#
+# == About
+#
+# The Ruby +Dir+ module is rubbish.  Time to clean it up a bit!
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+# 
+
+
+class Dir
+
+  # Return the absolute paths of files and directories in the
+  # directory, leaving out `.' and `..' entries.
+  def abs_contents
+    self.entries.map {|entry| File.join(self.path,entry) unless entry == '.' || entry == '..'}.compact
+  end
+end
+
+# puts "#{File.basename(__FILE__)}: You open the folder and see along list of names.  Some have been crossed out -- ominously..." # at bottom

data/lib/imw/utils/extensions/file_core.rb

@@ -0,0 +1,64 @@
+#
+# h2. lib/imw/utils/extensions/file.rb -- extensions to built-in file class
+#
+# == About
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+
+require 'imw/utils/error'
+require 'imw/utils/config'
+require 'imw/utils/extensions/string'
+
+class File
+
+  # Returns the name of the path given:
+  #
+  #   File.name_of_file("/path/to/somefile.txt") => "somefile".
+  def self.name_of_file path
+    basename(path)[0,basename(path).length - extname(path).length]
+  end
+
+  # Returns what would be the handle of a source or dataset
+  # described by a file at +path+:
+  #
+  #   File.handle "/path/to/a_particular_dataset.instructions.yaml"  #=> :a_particular_dataset
+  def self.handle path
+    File.basename(path).split('.').first.handle
+  end
+
+  # Returns a unique (non-existing) version of the given +path+ by
+  # appending successive intgers, useful for copying files ito
+  # directories without clobbering existing files (a la <tt>wget
+  # -nc</tt>).
+  #
+  # In a directory <tt>/path/to</tt> without a file named
+  # <tt>data.txt</tt>
+  #
+  #   File.uniquify("/path/to/data.txt") #=> "/path/to/data.txt"</tt>
+  #
+  # If <tt>data.txt</tt> were to already exist in that directory, then
+  #
+  #   File.uniquify("/path/to/data.txt") #=> "/path/to/data.txt.1"
+  #
+  # If <tt>data.txt.1</tt> were to already exist then
+  #
+  #   File.uniquify("/path/to/data.txt") #=> "/path/to/data.txt.2"
+  #
+  # and so on.
+  def self.uniquify path
+    orig_path = path.clone
+    copy_number = 1
+    while exist? path do
+      path = orig_path + ".#{copy_number}"
+      copy_number += 1
+    end
+    path
+  end
+
+end
+
+# puts "#{File.basename(__FILE__)}: You add a bit of glitter and jazz to all the folders in the cabinet.  It makes you feel happier when you have to sort through them." # at bottom