imw 0.2.0 → 0.2.1

data/lib/imw/{resources → compressed_files}/compressible.rb

@@ -10,9 +10,9 @@ module IMW
     :compress  => '',
     :extension => 'bz2'
   } unless defined?(COMPRESSION_SETTINGS)
-  
-  module Resources
 
+  module CompressedFiles
+    
     # Defines methods for compressing a file.  The default compression
     # program is defined in IMW::COMPRESSION_SETTINGS though a
     # particular resource can change the values in its
@@ -70,8 +70,6 @@ module IMW
           copy.mv(path) if copy.exist?
         end
       end
-      
     end
   end
 end
-

data/lib/imw/compressed_files/gz.rb

@@ -0,0 +1,16 @@
+module IMW
+  module CompressedFiles
+    module Gz
+
+      include IMW::CompressedFiles::Base
+      
+      def compression_settings
+        @compression_settings ||= {
+          :decompression_program  => :gunzip,
+          :decompress             => '-fd'
+        }
+      end
+      
+    end
+  end
+end

data/lib/imw/formats.rb

@@ -0,0 +1,31 @@
+module IMW
+  module Formats
+    autoload :Csv,   'imw/formats/delimited'
+    autoload :Tsv,   'imw/formats/delimited'
+    autoload :Excel, 'imw/formats/excel'
+    autoload :Json,  'imw/formats/json'
+    autoload :Xml,   'imw/formats/sgml'
+    autoload :Xsl,   'imw/formats/sgml'
+    autoload :Html,  'imw/formats/sgml'
+    autoload :Xhtml, 'imw/formats/sgml'
+    autoload :Rdf,   'imw/formats/sgml'      
+    autoload :Yaml,  'imw/formats/yaml'
+
+    # Handlers which augment a resource with data format specific
+    # methods.
+    HANDLERS = [
+                [ "Formats::Csv",   /\.csv$/    ],
+                [ "Formats::Tsv",   /\.tsv$/    ],
+                [ "Formats::Excel", /\.xslx?$/  ],
+                [ "Formats::Json",  /\.json$/   ],
+                [ "Formats::Xml",   /\.xml$/    ],
+                [ "Formats::Xsl",   /\.xsl$/    ],                         
+                [ "Formats::Html",  /\.html?$/  ],
+                [ "Formats::Xhtml", /\.xhtml?$/ ],
+                [ "Formats::Rdf",   /\.rdf?$/   ],
+                [ "Formats::Yaml",  /\.ya?ml$/  ]
+               ]
+  end
+end
+
+

data/lib/imw/formats/delimited.rb

@@ -0,0 +1,90 @@
+module IMW
+  module Formats
+
+    # Defines methods used for parsing and writing delimited data
+    # formats (CSV, TSV, &c.)  with the FasterCSV library.  This
+    # module is not used to directly extend a resource.  Instead,
+    # more specific modules (e.g. - IMW::Resources::Formats::Csv)
+    # include this one and also define +delimited_options+ which is
+    # actually what's passed to FasterCSV.
+    #
+    # @abstract
+    module Delimited
+
+      attr_accessor :delimited_settings
+
+      # Return the data in this delimited resource as an array of
+      # arrays.
+      #
+      # Yield each outer array (row) if passed a block.
+      #
+      # @return [Array] the full data matrix
+      # @yield [Array] each row of the data
+      def load &block
+        require 'fastercsv'
+        FasterCSV.parse(read, delimited_options, &block)
+      end
+
+      # Map each row in this delimited resource.
+      #
+      # @yield [Array] each row of the data
+      def map &block
+        load.map(&block)
+      end
+
+      # Dump an array of arrays into this resource.
+      #
+      # @param [Array] data array of arrays to dump
+      # @param [Hash] options
+      # @option options [true, false] :persist Keep this resource's IO object open after dumping
+      def dump data, options={}
+        require 'fastercsv'
+        data.each do |row|
+          write(FasterCSV.generate_line(row, delimited_options))
+        end
+        io.close unless options[:persist]
+        self
+      end
+    end
+
+    module Csv
+      include Delimited
+
+      # Default options to be passed to
+      # FasterCSV[http://fastercsv.rubyforge.org/]; see its
+      # documentation for more information.
+      #
+      # @return [Hash]
+      def delimited_options
+        @delimited_options ||= {
+          :col_sep        => ',',
+          :headers        => false,
+          :return_headers => false,
+          :write_headers  => true,
+          :skip_blanks    => false,
+          :force_quotes   => false
+        }
+      end
+    end
+
+    module Tsv
+      include Delimited
+
+      # Default options to be passed to
+      # FasterCSV[http://fastercsv.rubyforge.org/]; see its
+      # documentation for more information.
+      #
+      # @return [Hash]
+      def delimited_options
+        @delimited_options ||= {
+          :col_sep        => "\t",
+          :headers        => false,
+          :return_headers => false,
+          :write_headers  => true,
+          :skip_blanks    => false,
+          :force_quotes   => false
+        }
+      end
+    end
+  end
+end

data/lib/imw/formats/excel.rb

@@ -0,0 +1,125 @@
+module IMW
+  module Formats
+
+    # Defines methods for reading and writing Microsoft Excel data.
+    module Excel
+
+      attr_accessor :book, :sheet
+
+      def self.extended obj
+        if obj.exist?
+          @book = Spreadsheet.open path
+          @sheet = book.worksheet(0)
+          
+        end
+      end
+      
+
+      def book
+        return @book if @book
+        if exists?
+          @book = Spreadsheet.open(path)
+        else
+          @book = Spreadsheet::Workbook.new
+        end
+      end
+
+      def sheet
+        @sheet = @book.create_worksheet          
+        @sheet
+      end
+      
+      #If an Excel file exists at the location specified by uri then
+      #it is opened and can be read out with a subsequent call to
+      #load(). Otherwise, a new workbook is created and can be written
+      #to with the dump() method.
+      def initialize uri, mode='r', options={}
+        self.uri = uri
+        @max_lines = options[:max_lines] || 65000
+        @idx = 0
+        @book_idx = 0
+        @sht_idx = 0
+        unless self.exist?
+          make_new_book
+          make_new_sheet
+        else
+          get_existing_book
+        end
+      end
+
+      #Returns the data in an existing workbook as an
+      #array of arrays. Only capable of reading a single sheet.
+      def load
+        @sheet.map{|row| row.to_a}
+      end
+
+      #Dumps data, which is assumed to be an array of arrays, to a
+      #newly created Excel workbook. Attempting to dump to a book
+      #that already exists will typically result in file corruption.
+      #Raises a 'too many lines' error if the number of lines
+      #of data exceeds max_lines.
+      def dump data
+        data.each do |line|
+          raise "too many lines" if too_many?
+          self << line
+        end
+        save unless no_data?
+      end
+
+      #Processes a single line of data and updates internal variables.
+      #You shouldn't need to call this directly.
+      def << line
+        @sheet.row(@sht_row).concat( line )
+        @sht_row += 1
+        @idx += 1
+      end
+
+      #Instantiates a new Excel workbook in memory. You shouldn't
+      #need to call this directly.
+      def make_new_book
+        @book = Spreadsheet::Workbook.new
+        @book_idx += 1
+      end
+
+      #Makes a new worksheet for a pre-existing Excel workbook.
+      #This should be called after recovering from the
+      #'too many lines' error.
+      def make_new_sheet
+        @sheet = @book.create_worksheet
+        @sht_idx += 1
+        @sht_row = 0 #always start at row 0 in a new sheet
+      end
+
+      #Opens an existing Excel workbook. You shoudn't need to
+      #call this directly.
+      def get_existing_book
+        @sht_row = @sheet.row_count #would like to be able to dump new data, doesn't work
+        @sht_idx += 1
+      end
+
+      #Increments the current sheet to the next one in
+      #an open book. Not necessary at the moment.
+      def incr_sheet
+        @sheet = book.worksheet @sht_idx
+      end
+
+      #There are too many lines if the number of rows attempting
+      #to be written exceeds max_lines.
+      def too_many?
+        @sht_row >= @max_lines
+      end
+
+      #There is no data if the number of rows attempting to be written
+      #is zero.
+      def no_data?
+        @sht_row == 0
+      end
+
+      #Saves the workbook.
+      def save
+        @book.write path
+      end
+    end
+  end
+end
+  

data/lib/imw/formats/json.rb

@@ -0,0 +1,51 @@
+module IMW
+  module Formats
+
+    # Defines methods for reading and writing JSON data.
+    module Json
+
+      # Return the content of this resource.
+      #
+      # Will try to be smart about iterating over the data when
+      # passed a block.
+      #
+      # - if the outermost JSON data structure is an array, then
+      #   yield each element
+      #
+      # - if the outermost JSON data structure is a mapping, then
+      #   yield each key, value pair
+      #
+      # - otherwise just yield the structure
+      #
+      # @return [Hash, Array, String, Fixnum] whatever the JSON contained
+      def load &block
+        require 'json'
+        json = JSON.parse(read)
+        if block_given?
+          case json
+          when Array
+            json.each      { |obj| yield obj }
+          when Hash
+            json.each_pair { |key, value| yield key, value }
+          else
+            yield json
+          end
+        else
+          json
+        end
+      end
+
+      # Dump the +data+ into this resource.  It must be opened for
+      # writing.
+      #
+      # @param [Hash, String, Array, Fixnum] data the Ruby object to dump
+      # @option options [true, false] :persist (false) Don't close the IO object after writing
+      def dump data, options={}
+        require 'json'
+        write(data.to_json)
+        io.close unless options[:persist]
+        self
+      end
+    end
+  end
+end

data/lib/imw/formats/sgml.rb

@@ -0,0 +1,69 @@
+module IMW
+  module Formats
+
+    # Defines methods to parse SGML-derived data formats (XML, HTML,
+    # &c.).  This module isn't directly used to extend resources.
+    # Instead, more specific modules (e.g. -
+    # IMW::Resources::Formats::Xml) are used.
+    module Sgml
+
+      # Parse this resource using Hpricot and return (or yield if
+      # given a block) the resulting Hpricot::Doc.
+      #
+      # @return [Hpricot::Doc]
+      # @yield [Hpricot::Doc]
+      def load &block
+        require 'hpricot'
+        sgml = Hpricot(io)
+        if block_given?
+          yield sgml
+        else
+          sgml
+        end
+      end
+
+      # Parse the Hpricot::Doc of this resource with the given
+      # +parser+.
+      #
+      # The parser can either be an IMW::Parsers::HtmlParser or a
+      # hash which will be used to build such a parser.  See the
+      # documentation for IMW::Parsers::HtmlParser for more
+      # information.
+      #
+      # @param [Hash, IMW::Parsers::HtmlParser] parser
+      # @return [Hash] the parser's output
+      def parse parser
+        if parser.is_a?(IMW::Parsers::HtmlParser)
+          parser.parse(load)
+        else
+          IMW::Parsers::HtmlParser.new(parser).parse(load)
+        end
+      end
+    end
+
+    # Defines methods for XML data.
+    module Xml
+      include Sgml
+    end
+
+    # Defines methods for XSL data.
+    module Xsl
+      include Sgml
+    end
+
+    # Defines methods for XHTML data.
+    module Xhtml
+      include Sgml
+    end
+
+    # Defines methods for HTML data.
+    module Html
+      include Sgml
+    end
+
+    # Defines methods for RDF data.
+    module Rdf
+      include Sgml
+    end
+  end
+end

data/lib/imw/formats/yaml.rb

@@ -0,0 +1,51 @@
+module IMW
+  module Formats
+
+    # Provides methods for reading and writing YAML data.
+    module Yaml
+
+      # Return the content of this resource.
+      #
+      # Will try to be smart about iterating over the data when
+      # passed a block.
+      #
+      # - if the outermost YAML data structure is an array, then
+      #   yield each element
+      #
+      # - if the outermost YAML data structure is a mapping, then
+      #   yield each key, value pair
+      #
+      # - otherwise just yield the structure
+      #
+      # @return [Hash, Array, String, Fixnum] whatever the YAML contained
+      def load &block
+        require 'yaml'
+        yaml = YAML.load(read)
+        if block_given?
+          case yaml
+          when Array
+            yaml.each      { |obj| yield obj }
+          when Hash
+            yaml.each_pair { |key, value| yield key, value }
+          else
+            yield yaml
+          end
+        else
+          yaml
+        end
+      end
+
+      # Dump the +data+ into this resource.  It must be opened for
+      # writing.
+      #
+      # @param [Hash, String, Array, Fixnum] data the Ruby object to dump
+      # @option options [true, false] :persist (false) Don't close the IO object after writing
+      def dump data, options={}
+        require 'yaml'
+        write(data.to_yaml)
+        io.close unless options[:persist]
+        self
+      end
+    end
+  end
+end