imw 0.1.0

data/lib/imw/files/compressible.rb

@@ -0,0 +1,103 @@
+#
+# h2. lib/imw/files//compressible.rb -- compression module
+#
+# == About
+#
+# Module used for compression of files.  An including
+# <tt>IMW::Files::BasicFile</tt> object gains +compress+ and
+# <tt>compress!</tt> methods.
+#
+# By default, bzip2 is used for compression though gzip can also be
+# specified (the full list of known compression programs is in
+# <tt>IMW::Files::Compressible::COMPRESSION_PROGS</tt>).  Zip and Rar
+# compression are handled by the <tt>IMW::Files::Archive</tt> module.
+#
+# Decompression should be handled via the
+# <tt>IMW::Files::CompressedFile</tt> class.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+# 
+# puts "#{File.basename(__FILE__)}: Why is it that when you squeeze a lemon you get lemonade but when you squeeze a banana you just get a mess?" # at bottom
+module IMW
+  module Files
+    module Compressible
+
+      # Known compression programs.
+      COMPRESSION_PROGS = [:bzip2, :gzip]
+
+      # Extensions that are appended by each compression program.
+      COMPRESSION_EXTS = {
+        :bzip2 => '.bz2',
+        :gzip => '.gz'
+      }
+
+      # Compression flags for each program
+      COMPRESSION_FLAGS = {
+        :bzip2 => "-f",
+        :gzip => "-f"
+      }
+
+      protected
+      # Check that +program+ is a valid compression program.
+      def ensure_valid_compression_program program
+        raise IMW::Error.new("#{program} is not a valid compression program (#{COMPRESSION_PROGS.join(' ,')}).") unless COMPRESSION_PROGS.include? program
+      end
+      
+      # Construct the command passed to the shell to compress this
+      # file using the given +program+.
+      def compression_command program
+        ensure_valid_compression_program program
+        [IMW::EXTERNAL_PROGRAMS[program],COMPRESSION_FLAGS[program],self.path].join ' '
+      end
+
+      # Return the object representing this file compressed with
+      # +program+.
+      def compressed_file_path program
+        ensure_valid_compression_program program
+        path = File.join(self.dirname,self.basename + COMPRESSION_EXTS[program])
+      end
+      
+      public
+      # Compress this file in its present directory using +program+,
+      # overwriting any existing compressed files and without saving
+      # the original file.  Returns an
+      # <tt>IMW::Files::CompressedFile</tt> object corresponding to
+      # the compressed file.
+      # 
+      # Options:
+      # 
+      # <tt>:program</tt> (<tt>:bzip2</tt>):: names the compression
+      # program from the choices in <tt>IMW::EXTERNAL_PROGRAMS</tt>.
+      def compress! program = :bzip2
+        raise IMW::PathError.new("cannot compress #{@path}, doesn't exist!") unless exist?
+        FileUtils.cd(@dirname) { IMW.system(self.compression_command(program)) }
+        IMW.open(self.compressed_file_path(program))
+      end
+
+      # Compress this file in its present directory, overwriting any
+      # existing compressed files while keeping the original file.
+      # Returns an <tt>IMW::Files::CompressedFile</tt> object
+      # corresponding to the compressed file.
+      #
+      # Options:
+      # 
+      # <tt>:program</tt> (<tt>:bzip2</tt>):: names the compression
+      # program from the choices in <tt>IMW::EXTERNAL_PROGRAMS</tt>.
+      def compress program = :bzip2
+        raise IMW::PathError.new("cannot compress #{@path}, doesn't exist!") unless exist?        
+        begin
+          FileUtils.cp(self.path,self.path + 'copy')
+          compress! program
+        ensure
+          FileUtils.mv(self.path + 'copy',self.path)
+        end
+        IMW.open(self.compressed_file_path(program))
+      end
+      
+    end
+  end
+end
+

data/lib/imw/files/csv.rb

@@ -0,0 +1,112 @@
+#
+# h2. lib/imw/files/csv.rb -- CSV, TSV files
+#
+# == About
+#
+# For "comma-separated value" (CSV) and "tab-separated value" (TSV)
+# files.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+# 
+# puts "#{File.basename(__FILE__)}: Something clever" # at bottom
+
+require 'fastercsv'
+module IMW
+  module Files
+
+    # A base class from which to subclass various types of tabular
+    # data files (CSV, TSV, &c.)
+    class TabularDataFile < FasterCSV
+
+      include IMW::Files::BasicFile
+      include IMW::Files::Compressible
+      
+      # Default options to be passed to
+      # FasterCSV[http://fastercsv.rubyforge.org/]; see its
+      # documentation for more information.
+      DEFAULT_OPTIONS = {
+        :col_sep        => ',',
+        :headers        => false,
+        :return_headers => false,
+        :write_headers  => true,
+        :skip_blanks    => false,
+        :force_quotes   => false
+      }
+      
+      def initialize uri, mode='r', options = {}
+        options.reverse_merge!(self.class::DEFAULT_OPTIONS)
+        self.uri= uri
+        super open(uri,mode),options
+      end
+
+      # Return the contents of this CSV file as an array of arrays.
+      def load
+        entries
+      end
+
+      # Dump +data+ to this file.
+      #
+      # Options include:
+      # <tt>:flush</tt> (true):: flush the file buffer, writing it to disk
+      # <tt>:close</tt> (true):: close the file after writing +data+
+      def dump data, options = {}
+        options = options.reverse_merge :close => true, :flush => true
+        data.each {|row| self << row}
+        self.flush if options[:flush]
+        self.close if options[:close]
+        self
+      end
+
+      # Return a random sample of rows.
+      def sample length=10
+        rows, indices = [], Set.new
+        begin
+          each_with_index do |row, index|
+            break if rows.size == length
+            next if index != 0 && rand < 0.75   # skip 3/4 of rows after the 1st
+            rows    << row
+            indices << index
+          end
+          # now fill up to length if not there already
+          while rows.length < length
+            each_with_index do |row, index|
+              break if rows.size == length
+              next if index indices.include?(index)
+              rows << row
+            end
+          end
+          rows
+        rescue FasterCSV::MalformedCSVError
+          rows
+        end
+      end
+    end
+    
+
+    # Represents a file of comma-separated values (CSV).  This class
+    # is a subclass of <tt>FasterCSV</tt> so the methods of that
+    # library are available for use.
+    #
+    # See <tt>IMW::Files::TabularDataFile</tt> for more complete
+    # documentation.
+    class Csv < TabularDataFile
+    end
+
+    # Represents a file of tab-separated values (TSV).  This class
+    # is a subclass of <tt>FasterCSV</tt> so the methods of that
+    # library are available for use.
+    #
+    # See <tt>IMW::Files::TabularDataFile</tt> for more complete
+    # documentation.
+    class Tsv < TabularDataFile
+      DEFAULT_OPTIONS = {:col_sep => "\t"}.reverse_merge DEFAULT_OPTIONS
+    end
+
+    FILE_REGEXPS << [/\.csv$/, IMW::Files::Csv]
+    FILE_REGEXPS << [/\.tsv$/, IMW::Files::Tsv]
+
+  end
+end

data/lib/imw/files/json.rb

@@ -0,0 +1,41 @@
+
+# h2. lib/imw/files/json.rb -- describes json files
+#
+# == About
+#
+# A class for working with JSON files.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+#
+# puts "#{File.basename(__FILE__)}: Yet another clever comment." # at bottobm
+
+require 'json'
+require 'imw/files/text'
+
+module IMW
+  module Files
+
+    class Json < IMW::Files::Text
+
+      def initialize uri, mode='r', options = {}
+        super uri, mode
+      end
+
+      # Return the contents of this JSON file.
+      #
+      # FIXME what to do if a block is passed in?
+      def load &block
+        JSON.parse File.new(@path).read
+      end
+
+      # Dump +data+ to this file as JSON.
+      def dump data
+        super data.to_json
+      end
+    end
+    FILE_REGEXPS << [/\.json$/, IMW::Files::Json]
+  end
+end

data/lib/imw/files/sgml.rb

@@ -0,0 +1,65 @@
+#
+# h2. lib/imw/files/sgml.rb -- SGML files
+#
+# == About
+#
+# For SGML-derived files, including XML, HTML, &c..
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+# 
+# puts "#{File.basename(__FILE__)}: Something clever" # at bottom
+
+require 'hpricot'
+require 'imw/files/text'
+require 'imw/parsers/html_parser'
+
+module IMW
+  module Files
+
+    module Sgml
+
+      attr_accessor :doc
+
+      def initialize uri, mode='r', options={}
+        super uri, mode, options
+        @doc = Hpricot(open(uri))
+      end
+
+      # Delegate to Hpricot
+      def method_missing method, *args, &block
+        @doc.send method, *args, &block
+      end
+
+      # Parse this file using the IMW HTMLParser.  The parser can
+      # either be passed in directly or constructed from a passed hash
+      # of matchers.
+      def parse *args
+        parser = args.first.is_a?(IMW::HTMLParser) ? args.first : IMW::HTMLParser.new(*args)
+        parser.parse(self)
+      end
+
+    end
+
+    class Xml < IMW::Files::Text
+      include Sgml
+      def initialize uri, mode='r', options={}
+        super uri, mode, options
+        @doc = Hpricot.XML(open(uri))
+      end
+    end
+    
+    class Html < IMW::Files::Text
+      include Sgml
+      def initialize uri, mode='r', options={}
+        super uri, mode, options
+        @doc = Hpricot(open(uri))
+      end
+    end
+  end
+end
+
+
+

data/lib/imw/files/text.rb

@@ -0,0 +1,68 @@
+module IMW
+  module Files
+    
+    # Used to process text files when no more specialized class is suitable.
+    #
+    #   f = IMW::Files::Text.new '/path/to/my_file.dat'
+    #   f.load do |line|
+    #     # ...
+    #   end
+    #
+    # Missing methods will be passed to the associated file handle
+    # (either IO or StringIO depending on whether the URI passed in
+    # was local or remote) so the usual stuff like read or each_line
+    # still works.
+    class Text
+
+      include IMW::Files::BasicFile
+      include IMW::Files::Compressible
+
+      attr_reader :file, :parser
+
+      def initialize uri, mode='r', options = {}
+        self.uri= uri
+        raise IMW::PathError.new("Cannot write to remote file #{uri}") if mode == 'w' && remote?
+        @file = open(uri, mode)
+      end
+
+      # Return the contents of this text file as a string.
+      def load
+        file.read
+      end
+
+      # Return an array with each line of this file.  If given a
+      # block, pass each line to the block.
+      def entries &block
+        if block_given?
+          file.each do |line|
+            yield line.chomp
+          end
+        else
+          file.map do |line|
+            line.chomp
+          end
+        end
+      end
+
+      # Dump +data+ to this file as a string.  Close the file handle
+      # if passed in :close.
+      def dump data, options={}
+        file.write(data.inspect)
+        file.close if options[:close]
+      end
+
+      def method_missing method, *args
+        file.send method, *args
+      end
+
+      def parse parser_spec, &block
+        lines = parser_spec.delete(:lines)
+        @parser = IMW::Parsers::RegexpParser.new(parser_spec)
+        parser.parse!(file, {:lines => lines}, &block)
+      end
+      
+    end
+  end
+end
+
+# puts "#{File.basename(__FILE__)}: Don't forget to put a nametag on your Monkeywrench or one of the other chimps might steal it!" # at bottom

data/lib/imw/files/yaml.rb

@@ -0,0 +1,46 @@
+#
+# h2. lib/imw/files/yaml.rb -- describes yaml files
+#
+# == About
+#
+# A class for working with YAML files.
+#
+# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
+# Copyright:: Copyright (c) 2008 infochimps.org
+# License::   GPL 3.0
+# Website::   http://infinitemonkeywrench.org/
+# 
+
+require 'yaml'
+require 'imw/files/text'
+
+module IMW
+  module Files
+
+    class Yaml < IMW::Files::Text
+
+      def initialize uri, mode='r', options = {}
+        super uri, mode
+      end
+
+      # Return the contents of this YAML file.
+      #
+      # FIXME what to do if a block is passed in?
+      def load &block
+        YAML.load_file @path
+      end
+
+      # Dump +data+ to this file as YAML.
+      def dump data
+        super data.to_yaml
+      end
+      
+    end
+
+    FILE_REGEXPS << [/\.yaml$/, IMW::Files::Yaml]
+    FILE_REGEXPS << [/\.yml$/,  IMW::Files::Yaml]
+    
+  end
+end
+
+# puts "#{File.basename(__FILE__)}: Yet another clever comment." # at bottobm

data/lib/imw/packagers.rb

@@ -0,0 +1,8 @@
+module IMW
+  module Packagers
+    autoload :Archiver, 'imw/packagers/archiver'
+    autoload :S3Mover,  'imw/packagers/s3_mover'
+  end
+end
+
+