imw 0.1.1 → 0.2.0

data/lib/imw/files/json.rb

@@ -1,41 +0,0 @@
-
-# h2. lib/imw/files/json.rb -- describes json files
-#
-# == About
-#
-# A class for working with JSON files.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-# puts "#{File.basename(__FILE__)}: Yet another clever comment." # at bottobm
-
-require 'json'
-require 'imw/files/text'
-
-module IMW
-  module Files
-
-    class Json < IMW::Files::Text
-
-      def initialize uri, mode='r', options = {}
-        super uri, mode
-      end
-
-      # Return the contents of this JSON file.
-      #
-      # FIXME what to do if a block is passed in?
-      def load &block
-        JSON.parse File.new(@path).read
-      end
-
-      # Dump +data+ to this file as JSON.
-      def dump data
-        super data.to_json
-      end
-    end
-    FILE_REGEXPS << [/\.json$/, IMW::Files::Json]
-  end
-end

data/lib/imw/files/sgml.rb

@@ -1,46 +0,0 @@
-require 'hpricot'
-require 'imw/files/text'
-require 'imw/parsers/html_parser'
-
-module IMW
-  module Files
-
-    module Sgml
-
-      attr_accessor :doc
-
-      # Delegate to Hpricot
-      def method_missing method, *args, &block
-        @doc.send method, *args, &block
-      end
-
-      # Parse this file using the IMW::Parsers::HtmlParser.  The
-      # parser can either be passed in directly or constructed from a
-      # passed hash of specs and/or matchers.
-      def parse *args
-        parser = args.first.is_a?(IMW::Parsers::HtmlParser) ? args.first : IMW::Parsers::HtmlParser.new(*args)
-        parser.parse(self)
-      end
-
-    end
-
-    class Xml < IMW::Files::Text
-      include Sgml
-      def initialize uri, mode='r', options={}
-        super uri, mode, options
-        @doc = Hpricot.XML(open(uri))
-      end
-    end
-    
-    class Html < IMW::Files::Text
-      include Sgml
-      def initialize uri, mode='r', options={}
-        super uri, mode, options
-        @doc = Hpricot(open(uri))
-      end
-    end
-  end
-end
-
-
-

data/lib/imw/files/text.rb

@@ -1,68 +0,0 @@
-module IMW
-  module Files
-    
-    # Used to process text files when no more specialized class is suitable.
-    #
-    #   f = IMW::Files::Text.new '/path/to/my_file.dat'
-    #   f.load do |line|
-    #     # ...
-    #   end
-    #
-    # Missing methods will be passed to the associated file handle
-    # (either IO or StringIO depending on whether the URI passed in
-    # was local or remote) so the usual stuff like read or each_line
-    # still works.
-    class Text
-
-      include IMW::Files::BasicFile
-      include IMW::Files::Compressible
-
-      attr_reader :file, :parser
-
-      def initialize uri, mode='r', options = {}
-        self.uri= uri
-        raise IMW::PathError.new("Cannot write to remote file #{uri}") if mode == 'w' && remote?
-        @file = open(uri, mode)
-      end
-
-      # Return the contents of this text file as a string.
-      def load
-        file.read
-      end
-
-      # Return an array with each line of this file.  If given a
-      # block, pass each line to the block.
-      def entries &block
-        if block_given?
-          file.each do |line|
-            yield line.chomp
-          end
-        else
-          file.map do |line|
-            line.chomp
-          end
-        end
-      end
-
-      # Dump +data+ to this file as a string.  Close the file handle
-      # if passed in :close.
-      def dump data, options={}
-        file.write(data.inspect)
-        file.close if options[:close]
-      end
-
-      def method_missing method, *args
-        file.send method, *args
-      end
-
-      def parse parser_spec, &block
-        lines = parser_spec.delete(:lines)
-        @parser = IMW::Parsers::RegexpParser.new(parser_spec)
-        parser.parse!(file, {:lines => lines}, &block)
-      end
-      
-    end
-  end
-end
-
-# puts "#{File.basename(__FILE__)}: Don't forget to put a nametag on your Monkeywrench or one of the other chimps might steal it!" # at bottom

data/lib/imw/files/yaml.rb

@@ -1,46 +0,0 @@
-#
-# h2. lib/imw/files/yaml.rb -- describes yaml files
-#
-# == About
-#
-# A class for working with YAML files.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-# 
-
-require 'yaml'
-require 'imw/files/text'
-
-module IMW
-  module Files
-
-    class Yaml < IMW::Files::Text
-
-      def initialize uri, mode='r', options = {}
-        super uri, mode
-      end
-
-      # Return the contents of this YAML file.
-      #
-      # FIXME what to do if a block is passed in?
-      def load &block
-        YAML.load_file @path
-      end
-
-      # Dump +data+ to this file as YAML.
-      def dump data
-        super data.to_yaml
-      end
-      
-    end
-
-    FILE_REGEXPS << [/\.yaml$/, IMW::Files::Yaml]
-    FILE_REGEXPS << [/\.yml$/,  IMW::Files::Yaml]
-    
-  end
-end
-
-# puts "#{File.basename(__FILE__)}: Yet another clever comment." # at bottobm

data/lib/imw/files.rb

@@ -1,125 +0,0 @@
-require 'uri'
-require 'open-uri'
-require 'imw/utils'
-require 'imw/files/basicfile'
-require 'imw/files/directory'
-require 'imw/files/archive'
-require 'imw/files/compressible'
-require 'imw/files/compressed_file'
-
-module IMW
-
-  # Parse +path+ and return an appropriate handler.  Pass in <tt>:write
-  # => true</tt> to open for writing.
-  #
-  #   IMW.open("/tmp/test.csv") # => IMW::Files::Csv("/tmp/test.csv')
-  #
-  #
-  def self.open path, options = {}, &block
-    if File.directory?(File.expand_path(path))
-      dir = Files::Directory.new(path)
-      yield dir if block_given?
-      dir
-    else
-      mode = options[:write] ? 'w' : 'r'
-      file = Files.file_class_for(path, options).new(path, mode, options)
-      yield file if block_given?
-      file
-    end
-  end
-
-  def self.open! path, options = {}, &block
-    self.open path, options.reverse_merge(:write => true)
-  end
-
-  module Files
-
-
-    # There is certainly a cleaner way to do this.
-    autoload :Text,   'imw/files/text'
-    autoload :Binary, 'imw/files/binary'
-    autoload :Yaml,   'imw/files/yaml'
-    autoload :Csv,    'imw/files/csv'
-    autoload :Json,   'imw/files/json'
-    autoload :Bz2,    'imw/files/compressed_files_and_archives'
-    autoload :Gz,     'imw/files/compressed_files_and_archives'
-    autoload :Tar,    'imw/files/compressed_files_and_archives'
-    autoload :Tarbz2, 'imw/files/compressed_files_and_archives'
-    autoload :Targz,  'imw/files/compressed_files_and_archives'
-    autoload :Rar,    'imw/files/compressed_files_and_archives'
-    autoload :Zip,    'imw/files/compressed_files_and_archives'
-    autoload :Xml,    'imw/files/sgml'
-    autoload :Html,   'imw/files/sgml'
-    autoload :Excel,  'imw/files/excel'
-
-
-    # An array used to match files to classes to handle them.  The
-    # first element of each array is the regexp and the second names
-    # the class to handle the file.
-    #
-    #  IMW::Files::EXTENSION_HANDLERS << [ /\.csv$/, :csv ] #=> IMW::Files::Csv
-    #  IMW::Files::EXTENSION_HANDLERS << [ /\.txt$/, "Text" ] #=> IMW::Files::Text
-    #  IMW::Files::EXTENSION_HANDLERS << [ /\.myclass%/, MyClass ] #=> MyClass
-    #
-    # Elements at the end of the array have greater precedence which
-    # allows, say, <tt>.tar.gz</tt> to be handled differently from
-    # <tt>.gz</tt>.
-    EXTENSION_HANDLERS = [
-                          [/\.txt$/,      :text],
-                          [/\.txt$/,      :text],
-                          [/\.dat$/,      :text],
-                          [/\.ascii$/,    :text],
-                          [/\.yaml$/,     :yaml],
-                          [/\.yml$/,      :yaml],
-                          [/\.csv$/,      :csv],
-                          [/\.tsv$/,      :tsv],
-                          [/\.json$/,     :json],
-                          [/\.bz2$/,      :bz2],
-                          [/\.gz$/,       :gz],
-                          [/\.tar\.bz2$/, :tarbz2],
-                          [/\.tbz2$/,     :tarbz2],
-                          [/\.tar\.gz$/,  :targz],
-                          [/\.tgz$/,      :targz],
-                          [/\.tar$/,      :tar],
-                          [/\.rar$/,      :rar],
-                          [/\.zip$/,      :zip],
-                          [/\.xml$/,      :xml],
-                          [/\.html$/,     :html],
-                          [/\.htm$/,      :html],
-                          [/\.xlsx?$/,    :excel]
-                         ]
-
-    SCHEME_HANDLERS = [
-                       [/http/, :html]
-                       ]
-
-    protected
-    def self.file_class_for path, options = {}
-      klass = options.delete(:as)
-
-      # try to choose klass from path extension if not already set
-      unless klass
-        EXTENSION_HANDLERS.reverse_each do |regexp, thing| # end has greater precedence
-          next unless regexp =~ path
-          klass = thing
-          break
-        end
-      end
-
-      # try to choose klass from uri scheme if not already set
-      unless klass
-        scheme = URI.parse(path).scheme
-        SCHEME_HANDLERS.reverse_each do |regexp, thing| # end has greater precedence
-          next unless regexp =~ scheme
-          klass = thing
-          break
-        end
-      end
-
-      # just stick with text if still not set
-      klass = :text unless klass
-
-      klass.is_a?(Class) ? klass : class_eval(klass.to_s.downcase.capitalize)
-    end
-  end
-end

data/lib/imw/packagers/archiver.rb

@@ -1,126 +0,0 @@
-module IMW
-  module Packagers
-    
-    # Packages an Array of input files into a single output archive.
-    # When the archive is extracted, all the input files given will be
-    # in a single directory with a chosen name.  The path to the output
-    # archive determines both the name of the archive and its type (tar,
-    # tar.bz2, zip, &c.).
-    # 
-    # If any of the input files are themselves archives, they will first
-    # be extracted, with only their contents winding up in the final
-    # directory (the file hierarchy of the archive will be preserved).
-    # If any of the input files are compressed, they will first be
-    # uncompressed before being added to the directory.
-    #
-    # Input files can be renamed by passing in a Hash instead of an
-    # Array.  Each key in this hash is the path to an input file and its
-    # value is the new basename to give it.  If the basename is +nil+
-    # then the original path's basename will be used.
-    class Archiver
-
-      attr_accessor :name, :inputs
-
-      def initialize name, inputs
-        @name   = name
-        add_inputs inputs
-      end
-
-      # FIXME Instead of requiring +new_inputs+ to be either an Array
-      # or Hash just iterate through whatever it is using +each+ and
-      # see if the iterate can be interpreted as a mapping between
-      # strings.
-      def add_inputs new_inputs
-        @inputs ||= {}
-        if new_inputs.is_a?(Array)
-          new_inputs.each do |input|
-            @inputs[File.expand_path(input)] = File.basename(input)
-          end
-        else
-          new_inputs.each_pair do |input, basename|
-            @inputs[File.expand_path(input)] = (basename || File.basename(input))
-          end
-        end
-      end
-
-      def errors
-        @errors ||= []      
-      end
-
-      def add_processing_error error
-        IMW.logger.warn error      
-        errors << error
-      end
-
-      def success?
-        errors.empty?
-      end
-
-      # A temporary directory to work in.  Its contents will
-      # ultimately consist of a directory named for the package
-      # containing all the input files.
-      def tmp_dir
-        @tmp_dir ||= File.join(IMW.path_to(:tmp_root, 'packager'), (Time.now.to_i.to_s + "-" + $$.to_s)) # guaranteed unique on a node
-      end
-
-      def clean!
-        FileUtils.rm_rf(tmp_dir)
-      end
-
-      # A directory which will contain all the content being packaged,
-      # including the contents of any archives that were included in
-      # the list of files to process.
-      def dir
-        @dir ||= File.join(tmp_dir, name.to_s)
-      end
-
-      # FIXME This needs to be made idempotent -- calling prepare
-      # twice should not do any work the second time (unless the user
-      # is insistent and passes a :force option -- or maybe use bang
-      # and not-bang versions of the method for this distinction).
-      def prepare!
-        FileUtils.mkdir_p dir unless File.exist?(dir)
-        inputs.each_pair do |path, basename|
-          new_path = File.join(dir, basename)
-          file = IMW.open(path, :as => IMW::Files.file_class_for(basename)) # file's original path is meaningless: RackMultipart20091203-958-1nkgc61-0
-          case
-          when file.archive?
-            FileUtils.cd(dir) do
-              file.extract
-            end
-          when file.compressed?
-            file.cp(new_path).decompress!
-          else
-            file.cp(new_path)
-          end
-        end
-      end
-
-      # Package the contents of the temporary directory to an archive
-      # at +output+ but return exceptions instead of raising them.
-      def package output, options={}
-        begin
-          package! output, options={}
-        rescue RuntimeError => e
-          return e
-        end
-      end
-
-      # Package the contents of the temporary directory to an archive
-      # at +output+.
-      def package! output, options={}
-        output = IMW.open(output)         if output.is_a?(String)
-        FileUtils.mkdir_p(output.dirname) unless File.exist?(output.dirname)        
-        output.rm!                        if output.exist?
-        FileUtils.cd(tmp_dir) do
-          temp_output = IMW.open(output.basename)
-          packaged_output = temp_output.create(name.to_s + '/*').mv(output.path)
-          temp_output.rm if temp_output.exist?
-          add_processing_error "Archiver: couldn't create archive #{output.path}" unless output.exists?
-        end
-        output
-      end
-    end
-  end
-end
-

data/lib/imw/packagers/s3_mover.rb

@@ -1,36 +0,0 @@
-require 'aws/s3'
-module IMW
-  module Packagers
-    class S3Mover
-
-      attr_reader   :last_response
-      attr_accessor :bucket_name
-
-      def initialize options={}
-        @bucket_name = options.delete(:bucket_name)
-        AWS::S3::Base.establish_connection!(options)
-      end
-
-      def success?
-        errors.empty?
-      end
-
-      def success?
-        last_response && last_response.response.class == Net::HTTPOK
-      end
-
-      def upload local_path, remote_path
-        begin
-          upload! local_path, remote_path
-        rescue RuntimeError => e
-          return e
-        end
-      end
-
-      def upload! local_path, remote_path
-        @last_response = AWS::S3::S3Object.store(remote_path, open(local_path), bucket_name)
-      end
-      
-    end
-  end
-end

data/lib/imw/packagers.rb

@@ -1,8 +0,0 @@
-module IMW
-  module Packagers
-    autoload :Archiver, 'imw/packagers/archiver'
-    autoload :S3Mover,  'imw/packagers/s3_mover'
-  end
-end
-
-

data/lib/imw/utils/components.rb

@@ -1,61 +0,0 @@
-#
-# h2. lib/imw/utils/components.rb -- define separate components of IMW
-#
-# == About
-#
-# Defines a hash <tt>IMW::COMPONENTS</tt> which keys component names
-# to the files to be required to implement each component and defines
-# methods to load these files.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-# puts "#{File.basename(__FILE__)}: Something clever" # at bottom
-
-require 'imw/utils/error'
-
-module IMW
-
-  # Defines IMW components and the files required by each.  Components
-  # can be accessed using <tt>IMW.load_components</tt> or
-  # <tt>IMW#imw_components</tt>.
-  COMPONENTS = {
-    :datamapper => ["imw/dataset/datamapper","imw/dataset/datamapper/time_and_user_stamps"],
-    :data_mapper => :datamapper,
-    :html_parser => "imw/parsers/html_parser",
-    :flat_file_parser => "imw/parsers/flat_file_parser",
-    :line_parser => "imw/parsers/line_parser",
-    :infochimps => ["imw/infochimps/infochimps_resource","imw/infochimps/icss"]
-  }
-
-  # Load components of IMW as needed,
-  #
-  #   IMW.load_components :datamapper, :flat_file_parser
-  def self.load_components *args
-    args.each do |component_name|
-      begin
-        require component_name.to_s
-      rescue LoadError
-        component = IMW::COMPONENTS[component_name]
-        raise IMW::Error.new("#{component_name} is an invalid IMW component.  See IMW::COMPONENTS.") unless component        
-        if component.is_a? Array then
-          IMW.load_components *component
-        else
-          IMW.load_components component
-        end
-      end
-    end
-  end
-
-  # Load components of IMW as needed,
-  #
-  #   include IMW
-  #   imw_components :datamapper, :flat_file_parser
-  def imw_components *args
-    IMW.load_components *args
-  end
-
-end
-

data/lib/imw/utils/config.rb

@@ -1,46 +0,0 @@
-#
-# h2. lib/imw/utils/config.rb -- configuration parsing
-#
-# == About
-#
-# IMW looks for configuration settings in the following places, in
-# order of increasing precedence:
-#
-#   1. Settings defined directly in this file.
-#
-#   2. From the <tt>etc/imwrc</tt> file in the IMW root directory.
-#
-#   3. From the <tt>.imwrc</tt> file in the user's home directory (the
-#      filename can be changed; see
-#      <tt>IMW::Config::USER_CONFIG_FILE_BASENAME</tt>).
-#
-#   4. From the file defined by the environment variable +IMWRC+ (the
-#      value can be changed; see
-#      <tt>IMW::Config::USER_CONFIG_FILE_ENV_VARIABLE</tt>
-#
-# Settings not found in one configuration location will be searched
-# for in locations of lesser precedence.
-#
-# *Note:* configuration files are plain Ruby code that will be directly
-# evaluated.
-#
-# Relevant settings include
-#
-# * interfaces with external programs (+tar+, +wget+, &c.)
-# * paths to directories where IMW reads/writes files
-# * correspondences between file extensions and IMW file classes
-#
-# For more detailed information, see the default configuration file,
-# <tt>etc/imwrc</tt>.
-#
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-
-module IMW
-end
-
-# puts "#{File.basename(__FILE__)}: You carefully adjust the settings on your Monkeywrench.  Glob-monsters: beware!!" # at bottom

data/lib/imw/utils/extensions/class/attribute_accessors.rb

@@ -1,8 +0,0 @@
-# :nodoc:
-# for when cattr_accessor is all you need
-#
-require 'active_support/core_ext/array/extract_options'
-class Array #:nodoc:
-  include ActiveSupport::CoreExtensions::Array::ExtractOptions
-end
-require 'active_support/core_ext/class/attribute_accessors'

data/lib/imw/utils/extensions/core.rb

@@ -1,27 +0,0 @@
-require 'imw/utils/extensions/string'
-require 'imw/utils/extensions/array'
-require 'imw/utils/extensions/hash'
-require 'imw/utils/extensions/dir'
-require 'imw/utils/extensions/struct'
-require 'imw/utils/extensions/symbol'
-require 'imw/utils/extensions/file_core'
-require 'active_support/core_ext/module/aliasing'
-require 'active_support/core_ext/object/blank'
-require 'active_support/core_ext/object/misc'
-#require 'active_support/core_ext/blank.rb'
-require 'imw/utils/extensions/class/attribute_accessors'
-# require 'ostruct'
-require 'set'
-
-module IMW
-  # A replacement for the standard system call which raises an
-  # IMW::SystemCallError if the command fails as well as printing the
-  # command appended to the end of <tt>error_message</tt>.
-  def self.system *commands
-    command = commands.flatten.join ' '
-    Kernel.system(command)
-    raise IMW::SystemCallError.new(command) unless $?.success?
-  end
-end
-
-

data/lib/imw/utils/extensions/dir.rb

@@ -1,24 +0,0 @@
-#
-# h2. lib/imw/utils/extensions/dir.rb -- directory extensions
-#
-# == About
-#
-# The Ruby +Dir+ module is rubbish.  Time to clean it up a bit!
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-# 
-
-
-class Dir
-
-  # Return the absolute paths of files and directories in the
-  # directory, leaving out `.' and `..' entries.
-  def abs_contents
-    self.entries.map {|entry| File.join(self.path,entry) unless entry == '.' || entry == '..'}.compact
-  end
-end
-
-# puts "#{File.basename(__FILE__)}: You open the folder and see along list of names.  Some have been crossed out -- ominously..." # at bottom