imw 0.1.1 → 0.2.0

data/lib/imw/files/compressed_files_and_archives.rb

@@ -1,334 +0,0 @@
-module IMW
-  module Files
-    
-    # A class to wrap a +tar+ archive.
-    #
-    # Creation, appending, listing, and extraction flags are stored in
-    # <tt>IMW::Files::Tar::DEFAULT_FLAGS</tt> and all are passed to
-    # the <tt>:tar</tt> program in <tt>IMW::EXTERAL_PROGRAMS</tt>.
-    class Tar
-      
-      include IMW::Files::BasicFile
-      include IMW::Files::Archive
-      include IMW::Files::Compressible
-      
-      # The default flags used creating, appending to, listing, and
-      # extracting a tar archive.
-      DEFAULT_FLAGS = {
-        :create  => "-cf",
-        :append  => "-rf",
-        :list    => "-tf",
-        :extract => "-xf",
-        :program => :tar
-      }
-      
-      def initialize uri, *args
-        self.uri= uri
-        @archive = {
-          :program       => DEFAULT_FLAGS[:program],
-          :create_flags  => DEFAULT_FLAGS[:create],
-          :append_flags  => DEFAULT_FLAGS[:append],
-          :list_flags    => DEFAULT_FLAGS[:list],
-          :extract_flags => DEFAULT_FLAGS[:extract]
-        }
-      end
-    end # Tar
-    
-    # A class to wrap a <tt>tar.gz</tt> archive.
-    #
-    # Creation, appending, listing, and extraction flags are stored in
-    # <tt>IMW::Files::Targz::DEFAULT_FLAGS</tt> and all are passed to
-    # the <tt>:tar</tt> program in <tt>IMW::EXTERAL_PROGRAMS</tt>.
-    class Targz
-      
-      include IMW::Files::BasicFile
-      include IMW::Files::Archive
-      include IMW::Files::CompressedFile
-      
-      # The default flags used creating, appending to, listing, and
-      # extracting a <tt>tar.gz</tt> archive.
-      DEFAULT_FLAGS = {
-        :decompression_program => :gzip,
-        :decompression_flags   => '-fd',
-        :archive_program       => :tar,
-        :archive_list_flags    => "-tf",
-        :archive_extract_flags => "-xzf"
-      }
-      
-      def initialize uri, *args
-        self.uri= uri
-        @compression = {
-          :program             => DEFAULT_FLAGS[:decompression_program],
-          :decompression_flags => DEFAULT_FLAGS[:decompression_flags]
-        }
-        @archive = {
-          :program       => DEFAULT_FLAGS[:archive_program],
-          :list_flags    => DEFAULT_FLAGS[:archive_list_flags],
-          :extract_flags => DEFAULT_FLAGS[:archive_extract_flags]
-        }
-      end
-      
-      # Returns the path of the file after decompression.
-      def decompressed_path
-        if /\.tar\.gz$/.match @path then
-          @path.gsub /\.tar\.gz$/, ".tar"
-        elsif /\.tgz$/.match @path then
-          @path.gsub /\.tgz$/, ".tar"
-        end
-      end
-
-      def self.extname path
-        if /\.tar\.gz$/.match path then
-          ".tar.gz"
-        elsif /\.tgz$/.match path then
-          ".tgz"
-        end
-      end
-      
-    end # Targz
-
-    # A class to wrap a <tt>tar.bz2</tt> archive.
-    #
-    # Creation, appending, listing, and extraction flags are stored in
-    # <tt>IMW::Files::Tarbz2::DEFAULT_FLAGS</tt> and all are passed to
-    # the <tt>:tar</tt> program in <tt>IMW::EXTERAL_PROGRAMS</tt>.
-    class Tarbz2
-
-      include IMW::Files::BasicFile
-      include IMW::Files::Archive
-      include IMW::Files::CompressedFile
-      
-      # The default flags used creating, appending to, listing, and
-      # extracting a <tt>tar.bz2</tt> archive.
-      DEFAULT_FLAGS = {
-        :decompression_program => :bzip2,
-        :decompression_flags => '-fd',
-        :archive_program => :tar,
-        :archive_create_flags => '-cf',
-        :archive_list_flags => "-tf",
-        :archive_extract_flags => "-xjf"
-      }
-
-      def self.extname path
-        if /\.tar\.bz2$/.match path then
-          ".tar.bz2"
-        elsif /\.tbz2$/.match path then
-          ".tbz2"
-        end
-      end
-      
-      def initialize uri, *args
-        self.uri= uri
-        @compression = {
-          :program => DEFAULT_FLAGS[:decompression_program],
-          :decompression_flags => DEFAULT_FLAGS[:decompression]
-        }
-        @archive = {
-          :program => DEFAULT_FLAGS[:archive_program],
-          :list_flags => DEFAULT_FLAGS[:archive_list_flags],
-          :extract_flags => DEFAULT_FLAGS[:archive_extract_flags],
-          :create_flags  => DEFAULT_FLAGS[:archive_create_flags]
-        }
-      end
-
-      # Returns the path of the file after decompression.
-      def decompressed_path
-        if /\.tar\.bz2$/.match @path then
-          @path.gsub /\.tar\.bz2$/, ".tar"
-        elsif /\.tbz2$/.match @path then
-          @path.gsub /\.tbz2$/, ".tar"
-        end
-      end
-
-      # Overrides default behvaior of IMW::Files::Archive#create to
-      # compress files after creating them.
-      def create paths, opts={}
-        opts = opts.reverse_merge({:force => false})
-        raise IMW::Error.new("An archive already exists at #{@path}.") if exist? and not opts[:force]
-        paths = [paths] if paths.class == String
-        IMW.system IMW::EXTERNAL_PROGRAMS[@archive[:program]], @archive[:create_flags], path_between_archive_and_compression, *paths
-        IMW.open(path_between_archive_and_compression).compress!(:bzip2)
-      end
-
-      protected
-      def path_between_archive_and_compression
-        File.join(dirname,name + '.tar')
-      end
-      
-    end # Tarbz2
-    
-    # A class to wrap a +rar+ archive.
-    #
-    # Creation, appending, listing, and extraction flags are stored in
-    # <tt>IMW::Files::Rar::DEFAULT_FLAGS</tt> and all are passed to
-    # the <tt>:rar</tt> program in <tt>IMW::EXTERAL_PROGRAMS</tt>.
-    class Rar
-      
-      include IMW::Files::BasicFile
-      include IMW::Files::Archive
-      
-      # The default flags used creating, appending to, listing, and
-      # extracting a rar archive.
-      DEFAULT_FLAGS = {
-        :create => "a -r -o+ -inul",
-        :append => "a -r -o+ -inul",
-        :list => "vb",
-        :extract => "x -o+ -inul"
-      }
-      
-      def initialize uri, *args
-        self.uri= uri
-        @archive = {
-          :program => :rar,
-          :create_flags => DEFAULT_FLAGS[:create],
-          :append_flags => DEFAULT_FLAGS[:append],
-          :list_flags => DEFAULT_FLAGS[:list],
-          :extract_flags => DEFAULT_FLAGS[:extract]
-        }
-      end
-    end # Rar
-
-    # A class to wrap a +zip+ archive.
-    #
-    # Creation, appending, listing, and extraction flags are stored in
-    # <tt>IMW::Files::Zip::DEFAULT_FLAGS</tt> and all are passed to
-    # the <tt>:zip</tt> and <tt>:unzip</tt> programs in
-    # <tt>IMW::EXTERAL_PROGRAMS</tt>.
-    class Zip
-      
-      include IMW::Files::BasicFile
-      include IMW::Files::Archive
-      
-      # The default flags used creating, appending to, listing, and
-      # extracting a zip archive.
-      DEFAULT_FLAGS = {
-        :create => "-q -r",
-        :append => "-q -g",
-        :list => "-l",
-        :extract => "-q -o",
-        :unarchiving_program => :unzip
-      }
-      
-      def initialize uri, *args
-        self.uri= uri
-        @archive = {
-          :program => :zip,
-          :create_flags => DEFAULT_FLAGS[:create],
-          :append_flags => DEFAULT_FLAGS[:append],
-          :list_flags => DEFAULT_FLAGS[:list],
-          :extract_flags => DEFAULT_FLAGS[:extract],
-          :unarchiving_program => DEFAULT_FLAGS[:unarchiving_program]
-        }
-      end
-      
-      # The `unzip' program outputs data in a very annoying format:
-      #
-      #     Archive:  data.zip
-      #       Length     Date   Time    Name
-      #      --------    ----   ----    ----
-      #         18510  07-28-08 15:58   data/4d7Qrgz7.csv
-      #          3418  07-28-08 15:41   data/7S.csv
-      #         23353  07-28-08 15:41   data/g.csv
-      #           711  07-28-08 15:58   data/g.xml
-      #          1095  07-28-08 15:41   data/L.xml
-      #          2399  07-28-08 15:58   data/mTAu9H3.xml
-      #           152  07-28-08 15:58   data/vaHBS2t5R.dat
-      #      --------                   -------
-      #         49638                   7 files
-      #
-      # which is parsed by this method.
-      def archive_contents_string_to_array string
-        rows = string.split("\n")
-        # ignore the first 3 lines of the output and also discared the
-        # last 2 (5 = 2 + 3)
-        file_rows = rows[3,(rows.length - 5)]
-        file_rows.map! do |row|
-          # discard extra whitespace before after main text
-          row.lstrip!.rstrip!
-          # split the remaining text at spaces...columns beyond the
-          # third are part of the filename and should be joined with a
-          # space again in case of a filename with a space
-          row.split(' ')[3,row.size].join(' ')
-        end
-        file_rows
-      end
-    end # Zip
-
-    # A class to wrap a <tt>gz</tt> compressed file.
-    #
-    # The decompressing flags are stored in
-    # <tt>IMW::Files::Gz::DEFAULT_FLAGS</tt> and all are passed to the
-    # <tt>:gzip</tt> program in <tt>IMW::EXTERAL_PROGRAMS</tt>.
-    class Gz
-
-      include IMW::Files::BasicFile
-      include IMW::Files::CompressedFile
-      
-      # The default flags used in extracting a <tt>gz</tt> file.
-      DEFAULT_FLAGS = {
-        :program => :gzip,
-        :decompression => '-fd'
-      }
-      
-      def initialize uri, *args
-        self.uri= uri
-        @compression = {
-          :program => DEFAULT_FLAGS[:program],
-          :decompression_flags => DEFAULT_FLAGS[:decompression]
-        }
-      end
-
-      def decompressed_path
-        @path.gsub /\.gz$/, ""
-      end
-    end # Gz
-    
-    # A class to wrap a <tt>bz2</tt> compressed file.
-    #
-    # The decompressing flags are stored in
-    # <tt>IMW::Files::Bz2::DEFAULT_FLAGS</tt> and all are passed to
-    # the <tt>:bzip2</tt> program in <tt>IMW::EXTERAL_PROGRAMS</tt>.
-    class Bz2
-      
-      include IMW::Files::BasicFile
-      include IMW::Files::CompressedFile
-      
-      # The default flags used in extracting a <tt>bz2</tt> file.
-      DEFAULT_FLAGS = {
-        :program => :bzip2,
-        :decompression => '-fd'
-      }
-      
-      def initialize uri, *args
-        self.uri= uri
-        raise IMW::Error.new("#{@extname} is not a valid extension for a bzip2 compressed file.") unless @extname == '.bz2'
-        @compression = {
-          :program => DEFAULT_FLAGS[:program],
-          :decompression_flags => DEFAULT_FLAGS[:decompression]
-        }
-      end
-      
-      # Returns the path of the file after decompression.
-      def decompressed_path
-        @path.gsub /\.bz2$/, ""
-      end
-    end # Bz2
-
-
-    # make sure that tar.bz2 precedes bz2 and so on...
-    FILE_REGEXPS << [/\.tar\.bz2$/, IMW::Files::Tarbz2]
-    FILE_REGEXPS << [/\.tbz2$/,     IMW::Files::Tarbz2]
-    
-    FILE_REGEXPS << [/\.tar\.gz$/,  IMW::Files::Targz]
-    FILE_REGEXPS << [/\.tgz$/,      IMW::Files::Targz]
-
-    FILE_REGEXPS << [/\.tar$/,      IMW::Files::Tar]    
-    FILE_REGEXPS << [/\.bz2$/,      IMW::Files::Bz2]    
-    FILE_REGEXPS << [/\.gz$/,       IMW::Files::Gz]    
-    FILE_REGEXPS << [/\.rar$/,      IMW::Files::Rar]
-    FILE_REGEXPS << [/\.zip$/,      IMW::Files::Zip]
-    
-  end # Files
-end # IMW
-
-

data/lib/imw/files/compressible.rb

@@ -1,103 +0,0 @@
-#
-# h2. lib/imw/files//compressible.rb -- compression module
-#
-# == About
-#
-# Module used for compression of files.  An including
-# <tt>IMW::Files::BasicFile</tt> object gains +compress+ and
-# <tt>compress!</tt> methods.
-#
-# By default, bzip2 is used for compression though gzip can also be
-# specified (the full list of known compression programs is in
-# <tt>IMW::Files::Compressible::COMPRESSION_PROGS</tt>).  Zip and Rar
-# compression are handled by the <tt>IMW::Files::Archive</tt> module.
-#
-# Decompression should be handled via the
-# <tt>IMW::Files::CompressedFile</tt> class.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-# 
-# puts "#{File.basename(__FILE__)}: Why is it that when you squeeze a lemon you get lemonade but when you squeeze a banana you just get a mess?" # at bottom
-module IMW
-  module Files
-    module Compressible
-
-      # Known compression programs.
-      COMPRESSION_PROGS = [:bzip2, :gzip]
-
-      # Extensions that are appended by each compression program.
-      COMPRESSION_EXTS = {
-        :bzip2 => '.bz2',
-        :gzip => '.gz'
-      }
-
-      # Compression flags for each program
-      COMPRESSION_FLAGS = {
-        :bzip2 => "-f",
-        :gzip => "-f"
-      }
-
-      protected
-      # Check that +program+ is a valid compression program.
-      def ensure_valid_compression_program program
-        raise IMW::Error.new("#{program} is not a valid compression program (#{COMPRESSION_PROGS.join(' ,')}).") unless COMPRESSION_PROGS.include? program
-      end
-      
-      # Construct the command passed to the shell to compress this
-      # file using the given +program+.
-      def compression_command program
-        ensure_valid_compression_program program
-        [IMW::EXTERNAL_PROGRAMS[program],COMPRESSION_FLAGS[program],self.path].join ' '
-      end
-
-      # Return the object representing this file compressed with
-      # +program+.
-      def compressed_file_path program
-        ensure_valid_compression_program program
-        path = File.join(self.dirname,self.basename + COMPRESSION_EXTS[program])
-      end
-      
-      public
-      # Compress this file in its present directory using +program+,
-      # overwriting any existing compressed files and without saving
-      # the original file.  Returns an
-      # <tt>IMW::Files::CompressedFile</tt> object corresponding to
-      # the compressed file.
-      # 
-      # Options:
-      # 
-      # <tt>:program</tt> (<tt>:bzip2</tt>):: names the compression
-      # program from the choices in <tt>IMW::EXTERNAL_PROGRAMS</tt>.
-      def compress! program = :bzip2
-        raise IMW::PathError.new("cannot compress #{@path}, doesn't exist!") unless exist?
-        FileUtils.cd(@dirname) { IMW.system(self.compression_command(program)) }
-        IMW.open(self.compressed_file_path(program))
-      end
-
-      # Compress this file in its present directory, overwriting any
-      # existing compressed files while keeping the original file.
-      # Returns an <tt>IMW::Files::CompressedFile</tt> object
-      # corresponding to the compressed file.
-      #
-      # Options:
-      # 
-      # <tt>:program</tt> (<tt>:bzip2</tt>):: names the compression
-      # program from the choices in <tt>IMW::EXTERNAL_PROGRAMS</tt>.
-      def compress program = :bzip2
-        raise IMW::PathError.new("cannot compress #{@path}, doesn't exist!") unless exist?        
-        begin
-          FileUtils.cp(self.path,self.path + 'copy')
-          compress! program
-        ensure
-          FileUtils.mv(self.path + 'copy',self.path)
-        end
-        IMW.open(self.compressed_file_path(program))
-      end
-      
-    end
-  end
-end
-

data/lib/imw/files/csv.rb

@@ -1,113 +0,0 @@
-#
-# h2. lib/imw/files/csv.rb -- CSV, TSV files
-#
-# == About
-#
-# For "comma-separated value" (CSV) and "tab-separated value" (TSV)
-# files.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-# 
-# puts "#{File.basename(__FILE__)}: Something clever" # at bottom
-
-require 'fastercsv'
-module IMW
-  module Files
-
-    # A base class from which to subclass various types of tabular
-    # data files (CSV, TSV, &c.)
-    class TabularDataFile < FasterCSV
-
-      include IMW::Files::BasicFile
-      include IMW::Files::Compressible
-      
-      # Default options to be passed to
-      # FasterCSV[http://fastercsv.rubyforge.org/]; see its
-      # documentation for more information.
-      DEFAULT_OPTIONS = {
-        :col_sep        => ',',
-        :headers        => false,
-        :return_headers => false,
-        :write_headers  => true,
-        :skip_blanks    => false,
-        :force_quotes   => false
-      }
-      
-      def initialize uri, mode='r', options = {}
-        options.reverse_merge!(self.class::DEFAULT_OPTIONS)
-        self.uri= uri
-        options.delete(:write)  # FasterCSV complains about unkown options
-        super open(uri,mode), options
-      end
-
-      # Return the contents of this CSV file as an array of arrays.
-      def load
-        entries
-      end
-
-      # Dump +data+ to this file.
-      #
-      # Options include:
-      # <tt>:flush</tt> (true):: flush the file buffer, writing it to disk
-      # <tt>:close</tt> (true):: close the file after writing +data+
-      def dump data, options = {}
-        options = options.reverse_merge :close => true, :flush => true
-        data.each {|row| self << row}
-        self.flush if options[:flush]
-        self.close if options[:close]
-        self
-      end
-
-      # Return a random sample of rows.
-      def sample length=10
-        rows, indices = [], Set.new
-        begin
-          each_with_index do |row, index|
-            break if rows.size == length
-            next if index != 0 && rand < 0.75   # skip 3/4 of rows after the 1st
-            rows    << row
-            indices << index
-          end
-          # now fill up to length if not there already
-          while rows.length < length
-            each_with_index do |row, index|
-              break if rows.size == length
-              next if index indices.include?(index)
-              rows << row
-            end
-          end
-          rows
-        rescue FasterCSV::MalformedCSVError
-          rows
-        end
-      end
-    end
-    
-
-    # Represents a file of comma-separated values (CSV).  This class
-    # is a subclass of <tt>FasterCSV</tt> so the methods of that
-    # library are available for use.
-    #
-    # See <tt>IMW::Files::TabularDataFile</tt> for more complete
-    # documentation.
-    class Csv < TabularDataFile
-    end
-
-    # Represents a file of tab-separated values (TSV).  This class
-    # is a subclass of <tt>FasterCSV</tt> so the methods of that
-    # library are available for use.
-    #
-    # See <tt>IMW::Files::TabularDataFile</tt> for more complete
-    # documentation.
-    class Tsv < TabularDataFile
-      DEFAULT_OPTIONS = {:col_sep => "\t"}.reverse_merge DEFAULT_OPTIONS
-    end
-
-    FILE_REGEXPS << [/\.csv$/, IMW::Files::Csv]
-    FILE_REGEXPS << [/\.tsv$/, IMW::Files::Tsv]
-
-  end
-end

data/lib/imw/files/directory.rb

@@ -1,62 +0,0 @@
-require 'imw/files/basicfile'
-module IMW
-  module Files
-    class Directory
-
-      include IMW::Files::BasicFile
-
-      # FIXME these should be defined by BasicFile and then removed here but I don't see how...
-      # [:executable?, :executable_real?, :pipe?, :socket?, :rm, :rm!, :extname, :extname=, :name, :name=].each do |method|
-      #   instance_eval do
-      #     remove_method method
-      #   end
-      # end
-
-      def uri= uri
-        @uri      = uri.is_a?(URI::Generic) ? uri : URI.parse(uri)
-        @host     = self.uri.host
-        @path     = local? ? ::File.expand_path(self.uri.path) : self.uri.path
-        @dirname  = ::File.dirname path
-        @basename = ::File.basename path
-      end
-
-      def initialize uri
-        self.uri = uri
-      end
-
-      def [] selector='*'
-        Dir[File.join(path, selector)] if local?
-      end
-      
-      # Copy the contents of this directory to +new_dir+.
-      def cp new_dir
-        raise IMW::PathError.new("cannot copy from #{path}, doesn't exist!") unless exist?
-        if local?
-          FileUtils.cp_r path, new_dir
-        else
-          raise IMW::PathError.new("cannot recursively copy remote directories (yet!)")
-        end
-        self.class.new(new_dir)
-      end
-
-      # Move this directory to +new_dir+.
-      def mv new_dir
-        raise IMW::PathError.new("cannot move from #{path}, doesn't exist!") unless exist?
-        if local?
-          FileUtils.mv path, new_dir
-        else
-          raise IMW::PathError.new("cannot move remote directories (yet!)") 
-        end
-        self.class.new(new_dir)
-      end
-      alias_method :mv!, :mv
-
-      # Move this directory so it sits beneath +dir+.
-      def mv_to_dir dir
-        mv File.join(File.expand_path(dir),basename)
-      end
-      alias_method :mv_to_dir!, :mv_to_dir
-
-    end
-  end
-end

data/lib/imw/files/excel.rb

@@ -1,84 +0,0 @@
-require 'spreadsheet'
-
-# FIXME Main issue with this:
-# You can make a new excel book and dump data to it no problem.
-# However, something that doesn't seem to work is dumping to a file, opening,
-# and dumping to it again. At the moment this is probably not a big deal.
-
-
-module IMW
-  module Files
-    class Excel
-      include IMW::Files::BasicFile
-      include IMW::Files::Compressible
-
-      #need to initialize, load, and dump
-      attr_accessor :book,:idx, :max_lines, :sht_idx, :sht_row, :book_idx
-      def initialize uri, mode, options={}
-        self.uri = uri
-        @max_lines = options[:max_lines] || 65000
-        @idx = 0
-        @book_idx = 0
-        @sht_idx = 0
-        unless self.exist?
-          make_new_book
-          make_new_sheet
-        else
-          get_existing_book
-        end
-      end
-
-      def load
-        @sheet.map{|row| row.to_a}
-      end
-
-      def dump data
-        data.each do |line|
-          raise "too many lines" if too_many?
-          self << line
-        end
-        save unless no_data?
-      end
-
-      def << line
-        @sheet.row(@sht_row).concat( line )
-        @sht_row += 1
-        @idx += 1
-      end
-
-      def make_new_book
-        @book = Spreadsheet::Workbook.new
-        @book_idx += 1
-      end
-
-      def make_new_sheet
-        @sheet = @book.create_worksheet
-        @sht_idx += 1
-        @sht_row = 0 #always start at row 0 in a new sheet
-      end
-
-      def get_existing_book
-        @book = Spreadsheet.open path
-        @sheet = book.worksheet 0
-        @sht_row = @sheet.row_count #would like to be able to dump new data, doesn't work
-        @sht_idx += 1
-      end
-
-      def incr_sheet
-        @sheet = book.worksheet @sht_idx
-      end
-
-      def too_many?
-        @sht_row >= @max_lines
-      end
-
-      def no_data?
-        @sht_row == 0
-      end
-
-      def save
-        @book.write path
-      end
-    end
-  end
-end