rstore 0.2.0

data/lib/rstore/configuration.rb

@@ -0,0 +1,126 @@
+# encoding: utf-8
+require 'rstore/core_ext/object'
+
+module RStore
+  class Configuration
+
+    class << self
+      attr_reader :default_file_options
+      attr_reader :default_parse_options
+      attr_reader :default_options
+    end
+
+
+    # Supported options
+    @default_file_options    = {recursive: false, has_headers: true, selector: ''}
+    @default_parse_options   = {row_sep: :auto, col_sep: ",", quote_char: '"', field_size_limit: nil, skip_blanks: false}.freeze
+    @default_options         = @default_file_options.merge(@default_parse_options)
+
+
+    # Validations for RStore::CSV specific options
+    # @default_parse_options will not be validated here, as validation occurs on calling CSV.parse
+    Validations = Hash.new { |h,k| lambda { |value| true }}.
+      merge!({recursive:   lambda { |value| value.boolean_or_nil? },
+              has_headers: lambda { |value| value.boolean_or_nil? },
+              selector:    lambda { |value| value.is_a?(String) }})
+
+
+    attr_reader   :options
+    attr_reader   :file_options
+    attr_reader   :parse_options
+    attr_reader   :path
+
+
+    def initialize path, options
+      new_options  = options.dup
+
+      @path        = path
+      self.options = new_options
+      raise ArgumentError, arg_error_message(@path, new_options) if new_options.size > 0
+
+      @file_options  = extract_with(Configuration.default_file_options)
+      @parse_options = extract_with(Configuration.default_parse_options)
+    end
+
+
+    def options= options
+
+      new_options = Configuration.default_options.merge(options)
+
+      result = new_options.inject({}) do |acc, (option, value)|
+        if Configuration.default_options.include?(option)
+          if Configuration.valid_value?(option, value)
+            acc[option] = value
+            options.delete(option)
+          else
+            raise ArgumentError, "path #{@path}: '#{value}' (#{value.class}) is not a valid value for option #{option.inspect}"
+          end
+        end
+      acc
+      end
+
+      @options = result 
+    end
+
+
+    def extract_with options
+      keys = options.keys
+      @options.inject({}) do |acc, (option, value)|
+        if keys.include?(option)
+          acc[option] = value 
+        end
+
+      acc
+      end
+    end
+
+
+    def self.change_default_options options
+      raise ArgumentError, "#{options} must be an instance of Hash" unless options.is_a?(Hash)
+      new_options = Configuration.default_options.merge(options)
+      raise ArgumentError, "#{options} contains unknown option key" if new_options.size > Configuration.default_options.size
+      new_options.each do |option, value|
+        error_message = "'#{value}' (#{value.class}) is not a valid value for option #{option.inspect}"
+        raise ArgumentError, error_message unless valid_value?(option, value)
+      end
+
+      @default_options = new_options
+    end
+
+
+    def self.reset_default_options
+      @default_options = @default_file_options.merge(@default_parse_options)
+    end
+
+
+    def [] key
+      target = instance_variables.find do |var|
+        var.to_s.gsub(/@/,'').to_sym == key
+      end
+      if target
+        instance_variable_get(target)
+      else
+        raise ArgumentError, "'#{key}' is not an instance variable"
+      end
+    end
+
+
+    # Helper methods
+    # ------------------------------------------
+
+
+    def self.valid_value? option, value
+      Validations[option][value]
+    end
+
+
+    def arg_error_message path, new_options
+      keys = new_options.keys.join(', ')
+      "Unsupported options: #{keys} for path '#{path}'"
+    end
+
+
+  end
+end
+
+ 

data/lib/rstore/converter.rb

@@ -0,0 +1,144 @@
+# encoding: utf-8
+require 'rstore/logger'
+require 'bigdecimal'
+
+module RStore
+  class Converter
+
+    # @return [Date]
+    attr_reader   :data
+    # @return [Array<symbol>] Array of symbols representing the Ruby class set for each table column
+    attr_reader   :column_types
+    # @return [Array<Boolean>] Array of boolean values indicating if NULL is allowed as a column value
+    attr_reader   :allow_null
+    # @return [:symbol]
+    # On intitialization the only allowed value is :parsed.
+    # Will be set to :converted on successfull conversion.
+    attr_accessor :state
+
+    
+    boolean_converter = lambda do |field|
+      if field.downcase == 'true' || field == '1'
+        return true 
+      end
+      if field.downcase == 'false' || field == '0' 
+        return false
+      else
+        raise ArgumentError, "invalid value for Boolean() '#{field}'"
+      end
+    end
+
+    #  Converters used to verify the field data is valid. 
+    #  If a conversion fails, an exception is thrown together
+    #  with a descriptive error message pointing to the field 
+    #  where the error occured.
+    Converters = Hash.new {|h,k| h[k] = lambda { |field| field }}.
+      merge!({string:     lambda { |field| field },
+              date:       lambda { |field| Date.parse(field).to_s },
+              datetime:   lambda { |field| DateTime.parse(field).to_s }, 
+              # Convert to DateTime, because DateTime also checks if the argument is valid
+              time:       lambda { |field| DateTime.parse(field).to_s },
+              integer:    lambda { |field| Integer(field) },
+              float:      lambda { |field| Float(field) },
+              numeric:    lambda { |field| Float(field) },  # Handle Numeric as Float
+              # Check with Float first, then convert, because Float throws an error on invalid values such as 'x'.
+              bigdecimal: lambda { |field| Float(field); BigDecimal.new(field)},
+              boolean:    lambda { |field| boolean_converter[field] }})
+
+
+    def initialize data_object, database, table_name
+      state   = data_object.state
+      raise InvalidStateError, "#{state.inspect} is not a valid state for class Converter" unless state == :parsed
+      @data   = data_object.clone
+      @state  = @data.state
+      @schema = database.schema(table_name)
+      @column_types = extract_from_schema :type
+      @allow_null   = extract_from_schema :allow_null
+      @error  = false
+    end
+
+
+    def extract_from_schema target
+
+      schema = @schema.dup
+     
+      # Delete primary key column entry
+      schema.delete_if do |(_, property_hash)|
+        property_hash[:primary_key] == true
+      end
+
+      schema.map do |(_, property_hash)|
+        # Sequel handles Time as Datetime:
+        type = property_hash[target]
+        #type = (type == :time) ? :datetime : type
+        type
+      end
+    end
+
+
+    # Returns @table with converted fields if no error is thrown, nil otherwise
+    def convert
+      content = @data.content.dup
+
+      converted = content.each_with_index.map do |row, row_index|
+
+        convert_row(row, row_index)
+      end
+      @state = :converted 
+      Data.new(@data.path, converted, @state, @data.options)
+    end
+
+
+   
+    def convert_row row, row_index
+      # CSV.parse adjusts the size of each row to equal the size of the longest row 
+      # by adding nil where necessary.
+      error_message = <<-ERROR.gsub(/^\s+/,'')
+      Row length does not match number of columns. Please verify that:
+      1. The database table fits the csv table data
+      2. There is no primary key on a data column (you always need to 
+      define a separate column for an auto-incrementing primary key)
+      ERROR
+
+      raise InvalidRowLengthError, error_message unless row.size == @column_types.size
+
+      begin
+        row.each_with_index.map do |field, field_index|
+          @field_index = field_index
+
+          if field.nil?
+            validate_null(@allow_null[field_index], field)
+          else
+            convert_type(@column_types[field_index], field)
+          end
+        end
+      rescue ArgumentError, NullNotAllowedError => e
+        logger = Logger.new(@data)
+        logger.log(:convert, e, row: row_index, col: @field_index)
+        logger.error
+      end
+
+    rescue InvalidRowLengthError => e
+      logger = Logger.new(@data)
+      logger.log(:convert, e, row: row_index)
+      logger.error
+    end
+
+
+    # Helper methods ---------------------------------
+
+
+
+    def convert_type column_type, field
+      Converters[column_type][field]
+    end
+
+
+    def validate_null allow_null, field
+      raise NullNotAllowedError, "NULL value (empty field) not allowed" unless allow_null == true
+      field
+    end
+
+  end
+end
+ 

data/lib/rstore/core_ext/csv_wrapper.rb

@@ -0,0 +1,7 @@
+# encoding: utf-8
+
+require 'csv'
+
+# Wrapper around CSV to avoid name clashes inside RStore::CSV
+class CSVWrapper < CSV
+end

data/lib/rstore/core_ext/hash.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+class Hash
+
+  def include_pairs? hash
+    if hash.empty?
+      return false
+    else
+      hash.all? { |key, val| self[key] == val }
+    end
+  end
+
+end

data/lib/rstore/core_ext/object.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+class Object
+
+  def boolean_or_nil?
+    return true if self.nil?
+    !!self == self
+  end
+
+end

data/lib/rstore/core_ext/string.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+class String
+
+  def unquote
+    self.gsub(/(^"|"$)/,"").gsub(/""/,'"')
+  end
+
+  def is_i?
+    !!(self =~ /^[-+]?[0-9,]+$/)
+  end
+
+  # Checks if String represents a Float.
+  def is_f?
+    !!(self =~ /^[-+]?[0-9,]+\.[0-9]+$/)
+  end
+
+  def to_num
+    if self.is_f?
+      self.to_f
+    elsif self.is_i?
+      self.to_i
+    else
+    end
+  end
+
+
+  def url?
+    # http://daringfireball.net/2010/07/improved_regex_for_matching_urls
+    url_regex = /^((?:https?:\/\/|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}\/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+
+                   (?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’]))/x
+
+    !!(self =~ url_regex)
+
+  end
+
+
+
+
+
+
+
+end

data/lib/rstore/csv.rb

@@ -0,0 +1,288 @@
+# encoding: utf-8
+require 'open-uri'
+require 'rstore/data'
+require 'rstore/file_crawler'
+require 'rstore/converter'
+require 'rstore/configuration'
+require 'rstore/base_db'
+require 'rstore/base_table'
+require 'rstore/core_ext/string'
+
+
+module RStore
+  class CSV
+
+    #@return [BaseDB] a subclass of {RStore::BaseDB}
+    attr_reader :database
+    #@return [BaseTable] a sublcass of {RStore::BaseTable} 
+    attr_reader :table
+    #@return [Array<Data>] holds `RStore::Data` objects that are used internally to store information from a data source.
+    attr_reader :data_array
+    
+
+    # This constructor takes a block yielding an implicit instance of _self_.   
+    # Within the block, the following methods need to be called: 
+    #
+    # * {#from}
+    # * {#to}
+    # * {#run}
+    # @example  
+    #  RStore::CSV.new do
+    #    from '../easter/children', :recursive => true                   # select a directory or
+    #    from '../christmas/children/toys.csv'                           # file, or
+    #    from 'www.example.com/sweets.csv', :selector => 'pre div.line'  # URL
+    #    to   'company.products'                                         # provide database and table name
+    #    run                                                             # run the program
+    #  end
+    def initialize &block
+      @data_hash  = {}
+      @data_array = []
+      @database   = nil
+      @table      = nil
+
+      # Tracking method calls to #from, #to, and #run.
+      @from = false
+      @to   = false
+      @run  = false
+
+      instance_eval(&block) if block_given?
+
+    end
+
+
+    # Specify the source of the csv file(s) 
+    # There can be several calls to this method on given instance of `RStore::CSV`.
+    # This method has to be called before {#run}.
+    # @overload from(source, options)
+    #  @param [String] source The relative or full path to a directory, file, or an URL
+    #  @param [Hash] options The options used to customize fetching and parsing of csv data  
+    #  @option options [Boolean] :has_headers When set to false, the first line of a file is processed as data, otherwise it is discarded. 
+    #    (default: `true`)
+    #  @option options [Boolean] :recursive When set to true and a directory is given, recursively search for files. Non-csv files are skipped. 
+    #    (default: `false`]
+    #  @option options [String] :selector Mandatory css selector with an URL. Used the same syntax as Nokogiri, default: `""`
+    #  @option options [String] :col_sep The String placed between each field. (default: `","`)
+    #  @option options [String, Symbol] :row_sep The String appended to the end of each row. 
+    #    (default: `:auto`)
+    #  @option options [String] :quote_car The character used to quote fields.
+    #    (default: `'"'`)
+    #  @option options [Integer, Nil] :field_size_limit The maximum size CSV will read ahead looking for the closing quote for a field.
+    #    (default: `nil`)
+    #  @option options [Boolean] :skip_blanks When set to a true value, CSV will skip over any rows with no content.
+    #    (default: `false`) 
+    # @overload from(source)
+    #  @param [String] source The relative or full path to a directory, file, or an URL. The default options will be used.
+    # @return [void]
+    # @example
+    #  store = RStore::CSV.new
+    #  # fetching data from a file
+    #  store.from '../christmas/children/toys.csv'
+    #  # fetching data from a directory
+    #  store.from '../easter/children', :recursive => true
+    #  # fetching data from an URL
+    #  store.from 'www.example.com/sweets.csv', :selector => 'pre div.line'
+    def from source, options={}
+      crawler = FileCrawler.new(source, :csv, options)
+      @data_hash.merge!(crawler.data_hash)
+      @from = true
+    end
+
+
+    # Choose the database table to store the csv data into.
+    # This method has to be called before {#run}.
+    # @param [String] db_table The names of the database and table, separated by a dot, e.g. 'database.table'. 
+    #  The name of the database has to correspond to a subclass of `RStore::BaseDB`:
+    #  CompanyDB < RStore::BaseDB -> 'company'
+    #  The name of the table has to correspond to a subclass of `RStore::BaseTable`:
+    #  DataTable < RStore::BaseTable -> 'data'
+    # @return [void]
+    # @example
+    #  store = RStore::CSV.new
+    #  store.to('company.products')
+    def to db_table
+      @database, @table = CSV.database_table(db_table)
+      @to       = true
+    end
+
+
+    #@private
+    def self.database_table db_table
+      raise ArgumentError, "The name of the database and table have to be separated with a dot (.)"  unless delimiter_correct?(db_table)
+
+      db, tb = db_table.split('.')
+
+      database = BaseDB.db_classes[db.downcase.to_sym]
+      table    = BaseTable.table_classes[tb.downcase.to_sym]
+
+      raise Exception, "Database '#{db}' not found"  if database.nil?
+      raise Exception, "Table '#{tb}' not found"     if table.nil?
+
+      [database, table]
+    end
+
+
+    # Start processing the csv files, storing the data into a database table.
+    # Both methods, {#from} and {#to}, have to be called before this method.
+    # @return [void]
+    def run
+      return  if ran_once?   # Ignore subsequent calls to #run 
+      raise Exception, "At least one method 'from' has to be called before method 'run'"  unless @from == true
+      raise Exception, "Method 'to' has to be called before method 'run'"                 unless @to   == true
+
+      @data_hash.each do |path, data|
+        content = read_data(data)
+        @data_array << Data.new(path, content, :raw, data.options)  
+      end
+
+      @database.connect do |db|
+
+        create_table(db)
+        name = @table.name
+
+        prepared_data_array = @data_array.map do |data|
+          data.parse_csv.convert_fields(db, name)
+        end
+
+        insert_all(prepared_data_array, db, name)
+
+        @run = true
+        message = <<-TEXT.gsub(/^\s+/, '')
+        ===============================
+        All data has been successfully inserted into table '#{database.name}.#{table.name}'"
+        -------------------------------
+        You can retrieve all table data with the following code:
+        -------------------------------
+        #{self.class}.query('#{database.name}.#{table.name}') do |table|
+          table.all
+        end
+        ===============================
+        TEXT
+        puts message
+      end
+    end
+
+
+    #@private
+    def insert_all data_stream, database, name
+      database.transaction do  # outer transaction
+        data_stream.each do |data|
+          data.into_db(database, name)
+        end
+      end
+    end
+
+    private :insert_all
+
+
+    #@private
+    def read_data data_object
+      path    = data_object.path
+      options = data_object.options
+
+      begin
+        if path.url?
+          require 'nokogiri'
+          doc = Nokogiri::HTML(open(path))
+          selector = options[:file_options][:selector]
+
+          content = doc.css(selector).inject("") do |result, link|
+            result << link.content << "\n"
+            result
+          end
+        else
+          content = File.read(path)
+        end 
+
+      raise ArgumentError, "Empty content!"  if content.empty?
+
+      rescue Exception => e
+        logger = Logger.new(data_object)
+        logger.log(:fetch, e)
+        logger.error
+      end
+      
+      content
+    end
+
+
+    #@private
+    def create_table db
+
+      name = @table.name
+
+      if @database.connection_info.is_a?(Hash)
+        if @database.connection_info[:adapter] == 'mysql'
+          # http://sequel.rubyforge.org/rdoc/files/doc/release_notes/2_10_0_txt.html
+          Sequel::MySQL.default_engine = 'InnoDB'
+          # http://stackoverflow.com/questions/1671401/unable-to-output-mysql-tables-which-involve-dates-in-sequel
+          Sequel::MySQL.convert_invalid_date_time = nil 
+        end
+      end
+
+      unless db.table_exists?(name)
+        db.create_table(name, &@table.table_info)
+      end
+
+    end
+
+
+    # Easy querying by yielding a {http://sequel.rubyforge.org/rdoc/files/doc/dataset_basics_rdoc.html Sequel::Dataset} instance of your table.
+    # @param [String] db_table The name of the database and table, separated by a dot.
+    # @return [void]
+    # @yieldparam [Sequel::Dataset] table The dataset of your table
+    # @example
+    #  RStore::CSV.query('company.products') do |table|    # table = Sequel::Dataset object 
+    #    table.all                                         # fetch everything 
+    #    table.all[3]                                      # fetch row number 4 
+    #    table.filter(:id => 2).update(:on_stock => true)  # update entry
+    #    table.filter(:id => 3).delete                     # delete entry
+    #  end
+    def self.query db_table, &block
+      database, table = database_table(db_table)
+      database.connect do |db|
+        block.call(db[table.name]) if block_given?  # Sequel::Dataset
+      end
+    end
+
+    
+    
+    #@private
+    def self.delimiter_correct? name
+      !!(name =~ /^[^\.]+\.[^\.]+$/)
+    end
+
+    # Test if the data has been inserted into the database table.
+    # @return [Boolean]
+    def ran_once?
+      @run == true
+    end
+
+
+    # Change default options recognized by {#from}  
+    # The new option values apply to all following instances of `RStore::CSV`  
+    # Options can be reset to their defaults by calling {.reset_default_options}  
+    # See {#from} for a list of all options and their default values.  
+    # @param [Hash] options Keys from default options with their respective new values.  
+    # @return [void]
+    # @example
+    #   # Search directories recursively and handle the first row of a file as data by default
+    #   RStore::CSV.change_default_options(:recursive => true, :has_headers => false)
+    def self.change_default_options options
+      Configuration.change_default_options(options)
+    end
+
+   
+
+    # Reset the options recognized by {#from} to their default values.
+    # @return [void]
+    # @example 
+    #   RStore::CSV.reset_default_options
+    def self.reset_default_options
+      Configuration.reset_default_options
+    end
+
+
+  end
+end
+
+