imw 0.1.1 → 0.2.0

data/lib/imw/resources/archives_and_compressed/gz.rb

@@ -0,0 +1,18 @@
+module IMW
+  module Resources
+    module CompressedFiles
+      module Gz
+
+        include IMW::Resources::CompressedFile
+      
+        def compression_settings
+          @compression_settings ||= {
+            :decompression_program  => :gunzip,
+            :decompress             => '-fd'
+          }
+        end
+        
+      end
+    end
+  end
+end

data/lib/imw/resources/archives_and_compressed/rar.rb

@@ -0,0 +1,23 @@
+require 'imw/resources/archive'
+
+module IMW
+  module Resources
+    module Archives
+      module Rar
+        
+        include IMW::Resources::Archive
+        
+        def archive_settings
+          @archive_settings ||= {
+            :program => :rar,
+            :create  => ['a', '-o+', '-inul'],
+            :append  => ['a', '-o+', '-inul'],
+            :list    => "vb",
+            :extract => ['x', '-o+', '-inul']
+          }
+        end
+      end
+    end
+  end
+end
+

data/lib/imw/resources/archives_and_compressed/tar.rb

@@ -0,0 +1,23 @@
+require 'imw/resources/archive'
+
+module IMW
+  module Resources
+    module Archives
+      module Tar
+
+        include IMW::Resources::Archive
+        
+        def archive_settings
+          @archive_settings ||=  {
+            :create  => "-cf",
+            :append  => "-rf",
+            :list    => "-tf",
+            :extract => "-xf",
+            :program => :tar
+          }
+        end
+      end
+    end
+  end
+end
+

data/lib/imw/resources/archives_and_compressed/tarbz2.rb

@@ -0,0 +1,78 @@
+require 'imw/resources/archive'
+require 'imw/resources/compressed_file'
+
+module IMW
+  module Resources
+    module Archives
+      module Tarbz2
+
+        #
+        # It's a compressed file
+        #
+
+        include IMW::Resources::CompressedFile
+
+        def compression_settings
+          @compression_settings ||= {
+            :program               => :bzip2,
+            :decompression_program => :bunzip2,
+            :decompress            => '',
+            :extension             => 'bz2'
+          }
+        end
+
+        #
+        # But it's also an archive
+        #
+        
+        include IMW::Resources::Archive
+        
+        def archive_settings
+          @archive_settings ||= {
+            :program               => :tar,
+            :create                => '-cf',
+            :list                  => "-tjf",
+            :extract               => "-xjf"
+          }
+        end
+
+        # Overrides default behvaior of IMW::Files::Archive#create to
+        # compress files after creating them.
+        def create *input_paths
+          IMW.system(archive_settings[:program], archive_settings[:create], path_between_archive_and_compression, *input_paths.flatten)
+          IMW.open(path_between_archive_and_compression).compress!
+        end
+
+        def decompressed_basename
+          case extname
+          when '.tar.bz2' then basename[0..-5]               # .tar.bz2 => .tar
+          when '.tbz2'    then basename.gsub(/tbz2$/, 'tar') # .tbz2    => .tar
+          else                 basename[0..-(extname.size + 1)]
+          end
+        end
+        
+
+        protected
+        def path_between_archive_and_compression
+          File.join(dirname,name + '.tar')
+        end
+
+        public
+
+        #
+        # It's a compressed file AND an archive!
+        #
+        
+        def extname
+          case path
+          when /\.tar\.bz2$/ then '.tar.bz2'
+          when /\.tbz2$/     then '.tbz2'
+          else               File.extname(path)
+          end
+        end
+        
+      end
+    end
+  end
+end
+

data/lib/imw/resources/archives_and_compressed/targz.rb

@@ -0,0 +1,78 @@
+require 'imw/resources/archive'
+require 'imw/resources/compressed_file'
+
+module IMW
+  module Resources
+    module Archives
+      module Targz
+
+        #
+        # It's a compressed file
+        #
+
+        include IMW::Resources::CompressedFile
+
+        def compression_settings
+          @compression_settings ||= {
+            :program               => :gzip,
+            :decompression_program => :gunzip,
+            :decompress            => '',
+            :extension             => 'gz'
+          }
+        end
+
+        #
+        # But it's also an archive
+        #
+        
+        include IMW::Resources::Archive        
+        
+        def archive_settings
+          @archive_settings ||= {
+            :program               => :tar,        
+            :list                  => "-tzf",
+            :create                => '-cf',            
+            :extract               => "-xzf"
+          }
+        end
+
+        # Overrides default behvaior of IMW::Files::Archive#create to
+        # compress files after creating them.
+        def create *input_paths
+          IMW.system(archive_settings[:program], archive_settings[:create].split, path_between_archive_and_compression, *input_paths.flatten)
+          tar = IMW.open(path_between_archive_and_compression)
+          tar.compression_settings = compression_settings
+          tar.compress!
+        end
+
+        def decompressed_basename
+          case extname
+          when '.tar.gz' then basename[0..-4]              # .tar.gz => .tar
+          when '.tgz'    then basename.gsub(/tgz$/, 'tar') # .tgz    => .tar
+          else                basename[0..-(extname.size + 1)]
+          end
+        end
+        
+        protected
+        def path_between_archive_and_compression
+          File.join(dirname,name + '.tar')
+        end
+        public
+
+        #
+        # It's both an archive and a compressed file!
+        #
+
+        def extname
+          case path
+          when /\.tar\.gz$/ then '.tar.gz'
+          when /\.tgz$/     then '.tgz'
+          else              File.extname(path)
+          end
+        end
+
+      end
+    end
+  end
+end
+

data/lib/imw/resources/archives_and_compressed/zip.rb

@@ -0,0 +1,57 @@
+require 'imw/resources/archive'
+
+module IMW
+  module Resources
+    module Archives
+      module Zip
+
+        include IMW::Resources::Archive
+
+        def archive_settings
+          @archive_settings ||= {
+            :program             => :zip,
+            :create              => "-qqr",
+            :append              => "-qqg",
+            :list                => "-l",
+            :extract             => "-qqo",
+            :unarchiving_program => :unzip
+          }
+        end
+
+        protected
+        
+        # The `unzip' program outputs data in a very annoying format:
+        #
+        #     Archive:  data.zip
+        #       Length     Date   Time    Name
+        #      --------    ----   ----    ----
+        #         18510  07-28-08 15:58   data/4d7Qrgz7.csv
+        #          3418  07-28-08 15:41   data/7S.csv
+        #         23353  07-28-08 15:41   data/g.csv
+        #           711  07-28-08 15:58   data/g.xml
+        #          1095  07-28-08 15:41   data/L.xml
+        #          2399  07-28-08 15:58   data/mTAu9H3.xml
+        #           152  07-28-08 15:58   data/vaHBS2t5R.dat
+        #      --------                   -------
+        #         49638                   7 files            
+        #
+        # which is parsed by this method.
+        def archive_contents_string_to_array string
+          rows = string.split("\n")
+          # ignore the first 3 lines of the output and also discared the
+          # last 2 (5 = 2 + 3)
+          file_rows = rows[3,(rows.length - 5)]
+          file_rows.map do |row|
+            if row
+              columns = row.lstrip.rstrip.split(' ')
+              # grab the filename in the fourth column
+              columns[3..-1].join(' ')
+            end
+          end.compact
+        end
+      end
+    end
+  end
+end
+
+      

data/lib/imw/resources/archives_and_compressed.rb

@@ -0,0 +1,32 @@
+module IMW
+  module Resources
+    autoload :Compressible,    'imw/resources/compressible'
+    autoload :CompressedFile,  'imw/resources/compressed_file'
+    autoload :Archive,         'imw/resources/archive'
+    autoload :Archives,        'imw/resources/archive'
+    autoload :CompressedFiles, 'imw/resources/compressed_file'
+
+    # Handlers which augment the resource with methods for archiving,
+    # extracting, compressing, decompressing...
+    ARCHIVE_AND_COMPRESSED_HANDLERS = [
+
+                                       # try compressible first -- compressed files below will override it
+                                       ["Compressible",         Proc.new { |r| r.is_local? } ],
+
+                                       # order is important! -- tar.bz2 must come before .bz2, &c.
+                                       ["Archives::Tarbz2",     Proc.new { |r| r.is_local? && r.path =~ /\.tar\.bz2$/                                              } ],
+                                       ["Archives::Tarbz2",     Proc.new { |r| r.is_local? && r.path =~ /\.tbz2$/                                                  } ],
+                                       ["CompressedFiles::Bz2", Proc.new { |r| r.is_local? && r.path =~ /\.bz2$/ && r.path !~ /\.tar\.bz2$/ && r.path !~ /\.tbz2$/ } ],
+                                       ["Archives::Targz",      Proc.new { |r| r.is_local? && r.path =~ /\.tar\.gz$/                                               } ],
+                                       ["Archives::Targz",      Proc.new { |r| r.is_local? && r.path =~ /\.tgz$/                                                   } ],
+                                       ["CompressedFiles::Gz",  Proc.new { |r| r.is_local? && r.path =~ /\.gz$/  && r.path !~ /\.tar\.gz$/  && r.path !~ /\.tgz$/  } ],
+                                       ["Archives::Tar",        Proc.new { |r| r.is_local? && r.path =~ /\.tar$/                                                   } ],
+                                       ["Archives::Rar",        Proc.new { |r| r.is_local? && r.path =~ /\.rar$/                                                   } ],
+                                       ["Archives::Zip",        Proc.new { |r| r.is_local? && r.path =~ /\.zip$/                                                   } ]
+
+                                      ]
+
+    
+  end
+end
+    

data/lib/imw/resources/compressed_file.rb

@@ -0,0 +1,89 @@
+module IMW
+  module Resources
+    
+    module CompressedFiles
+      autoload :Bz2, 'imw/resources/archives_and_compressed/bz2'
+      autoload :Gz, 'imw/resources/archives_and_compressed/gz'
+    end
+
+    # Defines methods for decompressing a compressed file.  This
+    # module isn't used to directly extend an IMW::Resource --
+    # instead, format specific modules (e.g. -
+    # IMW::Resources::CompressedFiles::Bz2) include this module and
+    # further define the command-line flags &c. needed to make
+    # everything work.
+    module CompressedFile
+
+      attr_accessor :compression_settings
+
+      # Is this file compressed?
+      #
+      # @return [true, false]
+      def is_compressed?
+        true
+      end
+
+      # Can this file be compressed?
+      #
+      # @return [true, false]
+      def is_compressible?
+        false
+      end
+
+      # The basename of this resource after it is decompressed
+      #
+      #   IMW::Resource.new('/path/to/my_file.txt.bz2').decompressed_basename
+      #   => 'my_file.txt'
+      #
+      # @return [String] the decompressed basename
+      def decompressed_basename
+        basename[0..-(extname.size + 1)]
+      end
+      
+      # The path of this resource after it is decompressed
+      #
+      #   IMW::Resource.new('/path/to/my_file.txt.bz2').decompressed_basename
+      #   => '/path/to/my_file.txt'
+      #
+      # @return [String] the decompressed path
+      def decompressed_path
+        File.join(dirname, decompressed_basename)
+      end
+      
+      # Decompress this file in its present directory overwriting any
+      # existing files and without saving the original compressed
+      # file.
+      #
+      # @return [IMW::Resource] the decompressed resource
+      def decompress!
+        should_exist!("Cannot decompress.")
+        program = compression_settings[:decompression_program] || compression_settings[:program]
+        FileUtils.cd(dirname) { IMW.system(program, compression_settings[:decompress], path) }
+        IMW.open(decompressed_path)
+      end
+
+      # Decompress this file in its present directory, overwriting any
+      # existing files while keeping the original compressed file.
+      #
+      # FIXME The implementation is a little stupid as the file is
+      # needlessly copied.
+      #
+      # @return [IMW::Resource] the decompressed resource
+      def decompress
+        should_exist!("Cannot decompress.")
+        begin
+          copy = cp(path + '.imw_copy')
+          regular_file = decompress!
+          copy.mv(path)
+          regular_file
+        ensure
+          copy.mv(path) if copy && copy.exist?
+        end
+      end
+
+    end
+  end
+end
+
+
+

data/lib/imw/resources/compressible.rb

@@ -0,0 +1,77 @@
+module IMW
+
+  # Default settings used when compressing files.  <tt>:program</tt>
+  # defines the name of the command-line program to use,
+  # <tt>:compress</tt> gives the flags to use when compressing, and
+  # <tt>:extension</tt> gives the extension (_without_ the `.') added
+  # by the program after compressing.
+  COMPRESSION_SETTINGS = {
+    :program   => 'bzip2',
+    :compress  => '',
+    :extension => 'bz2'
+  } unless defined?(COMPRESSION_SETTINGS)
+  
+  module Resources
+
+    # Defines methods for compressing a file.  The default compression
+    # program is defined in IMW::COMPRESSION_SETTINGS though a
+    # particular resource can change the values in its
+    # +compression_settings+ hash.
+    module Compressible
+
+      # Compression settings.
+      attr_accessor :compression_settings
+
+      # Is this file compressible?
+      #
+      # @return [true]
+      def is_compressible?
+        true
+      end
+      
+      # Defines the compression settings used for this
+      # resource. <tt>:program</tt> defines the name of the
+      # command-line program to use, <tt>:compress</tt> gives the
+      # flags to use when compressing, and <tt>:extension</tt> gives
+      # the extension (_without_ the `.') added by the program after
+      # compressing.
+      #
+      # @return [Hash]
+      def compression_settings
+        @compression_settings ||= COMPRESSION_SETTINGS
+      end
+
+      # Compress this resource in place, overwriting it.
+      #
+      # This resource's +compression_settings+ method is used to
+      # determine the method of compression.
+      #
+      # @return [IMW::Resource] the compressed file
+      def compress!
+        should_exist!("Cannot compress.")
+        IMW.system(*[compression_settings[:program], compression_settings[:compress], path])
+        IMW.open(File.join(dirname,basename + "." + compression_settings[:extension]))
+      end
+
+      # Compress this resource without overwriting it.
+      #
+      # FIXME The implementation is a little stupid as the file is
+      # needlessly copied.
+      #
+      # @return [IMW::Resource] the compressed file
+      def compress options={}
+        should_exist!("Cannot compress.")
+        begin
+          copy = cp(path + '.imw_copy')
+          compressed_file = compress!
+          copy.mv(path)
+          compressed_file
+        ensure
+          copy.mv(path) if copy.exist?
+        end
+      end
+      
+    end
+  end
+end
+

data/lib/imw/resources/formats/delimited.rb

@@ -0,0 +1,92 @@
+module IMW
+  module Resources
+    module Formats
+
+      # Defines methods used for parsing and writing delimited data
+      # formats (CSV, TSV, &c.)  with the FasterCSV library.  This
+      # module is not used to directly extend a resource.  Instead,
+      # more specific modules (e.g. - IMW::Resources::Formats::Csv)
+      # include this one and also define +delimited_options+ which is
+      # actually what's passed to FasterCSV.
+      #
+      # @abstract
+      module Delimited
+
+        attr_accessor :delimited_settings
+
+        # Return the data in this delimited resource as an array of
+        # arrays.
+        #
+        # Yield each outer array (row) if passed a block.
+        #
+        # @return [Array] the full data matrix
+        # @yield [Array] each row of the data
+        def load &block
+          require 'fastercsv'
+          FasterCSV.parse(read, delimited_options, &block)
+        end
+
+        # Map each row in this delimited resource.
+        #
+        # @yield [Array] each row of the data
+        def map &block
+          load.map(&block)
+        end
+
+        # Dump an array of arrays into this resource.
+        #
+        # @param [Array] data array of arrays to dump
+        # @param [Hash] options
+        # @option options [true, false] :persist Keep this resource's IO object open after dumping
+        def dump data, options={}
+          require 'fastercsv'
+          data.each do |row|
+            write(FasterCSV.generate_line(row, delimited_options))
+          end
+          io.close unless options[:persist]
+          self
+        end
+      end
+
+      module Csv
+        include Delimited
+
+        # Default options to be passed to
+        # FasterCSV[http://fastercsv.rubyforge.org/]; see its
+        # documentation for more information.
+        #
+        # @return [Hash]
+        def delimited_options
+          @delimited_options ||= {
+            :col_sep        => ',',
+            :headers        => false,
+            :return_headers => false,
+            :write_headers  => true,
+            :skip_blanks    => false,
+            :force_quotes   => false
+          }
+        end
+      end
+
+      module Tsv
+        include Delimited
+
+        # Default options to be passed to
+        # FasterCSV[http://fastercsv.rubyforge.org/]; see its
+        # documentation for more information.
+        #
+        # @return [Hash]
+        def delimited_options
+          @delimited_options ||= {
+            :col_sep        => "\t",
+            :headers        => false,
+            :return_headers => false,
+            :write_headers  => true,
+            :skip_blanks    => false,
+            :force_quotes   => false
+          }
+        end
+      end
+    end
+  end
+end

data/lib/imw/resources/formats/excel.rb

@@ -0,0 +1,125 @@
+module IMW
+  module Resources
+    module Formats
+
+      # Defines methods for reading and writing Microsoft Excel data.
+      module Excel
+
+        attr_accessor :book, :sheet
+
+        def self.extended obj
+          if obj.exist?
+            @book = Spreadsheet.open path
+            @sheet = book.worksheet(0)
+            
+          end
+        end
+        
+
+        def book
+          return @book if @book
+          if exists?
+            @book = Spreadsheet.open(path)
+          else
+            @book = Spreadsheet::Workbook.new
+          end
+        end
+
+        def sheet
+          @sheet = @book.create_worksheet          
+          @sheet
+        end
+        
+        #If an Excel file exists at the location specified by uri then
+        #it is opened and can be read out with a subsequent call to
+        #load(). Otherwise, a new workbook is created and can be written
+        #to with the dump() method.
+        def initialize uri, mode='r', options={}
+          self.uri = uri
+          @max_lines = options[:max_lines] || 65000
+          @idx = 0
+          @book_idx = 0
+          @sht_idx = 0
+          unless self.exist?
+            make_new_book
+            make_new_sheet
+          else
+            get_existing_book
+          end
+        end
+
+        #Returns the data in an existing workbook as an
+        #array of arrays. Only capable of reading a single sheet.
+        def load
+          @sheet.map{|row| row.to_a}
+        end
+
+        #Dumps data, which is assumed to be an array of arrays, to a
+        #newly created Excel workbook. Attempting to dump to a book
+        #that already exists will typically result in file corruption.
+        #Raises a 'too many lines' error if the number of lines
+        #of data exceeds max_lines.
+        def dump data
+          data.each do |line|
+            raise "too many lines" if too_many?
+            self << line
+          end
+          save unless no_data?
+        end
+
+        #Processes a single line of data and updates internal variables.
+        #You shouldn't need to call this directly.
+        def << line
+          @sheet.row(@sht_row).concat( line )
+          @sht_row += 1
+          @idx += 1
+        end
+
+        #Instantiates a new Excel workbook in memory. You shouldn't
+        #need to call this directly.
+        def make_new_book
+          @book = Spreadsheet::Workbook.new
+          @book_idx += 1
+        end
+
+        #Makes a new worksheet for a pre-existing Excel workbook.
+        #This should be called after recovering from the
+        #'too many lines' error.
+        def make_new_sheet
+          @sheet = @book.create_worksheet
+          @sht_idx += 1
+          @sht_row = 0 #always start at row 0 in a new sheet
+        end
+
+        #Opens an existing Excel workbook. You shoudn't need to
+        #call this directly.
+        def get_existing_book
+          @sht_row = @sheet.row_count #would like to be able to dump new data, doesn't work
+          @sht_idx += 1
+        end
+
+        #Increments the current sheet to the next one in
+        #an open book. Not necessary at the moment.
+        def incr_sheet
+          @sheet = book.worksheet @sht_idx
+        end
+
+        #There are too many lines if the number of rows attempting
+        #to be written exceeds max_lines.
+        def too_many?
+          @sht_row >= @max_lines
+        end
+
+        #There is no data if the number of rows attempting to be written
+        #is zero.
+        def no_data?
+          @sht_row == 0
+        end
+
+        #Saves the workbook.
+        def save
+          @book.write path
+        end
+      end
+    end
+  end