rstore 0.2.0

data/lib/rstore/data.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+
+require 'csv'
+require 'rstore/converter'
+require 'rstore/storage'
+require 'rstore/core_ext/object'
+require 'rstore/core_ext/csv_wrapper'
+
+module RStore
+  class Data
+
+    attr_reader   :path
+    attr_reader   :content
+    attr_reader   :state
+    attr_reader   :options
+
+
+    KnownStates = [:raw, :parsed, :converted, :error]
+
+
+    def initialize path, content, state, options
+      error_message = "#{path}: The following options are not valid as an argument to #{self.class}:\n#{options}"
+      raise ArgumentError, error_message  unless options.is_a?(Hash) 
+      @path      = path
+      @content   = content
+      self.state = state
+      @options   = options
+    end
+
+  
+    #def extract_type path
+    #  path, filename = File.split(path)
+    #  filename.match(/\.(?<type>.*)$/)[:type].to_sym
+    #end
+
+    def parse_csv
+      raise InvalidStateError, "#{state.inspect} is not a valid Data state for method 'to_csv'"  unless state == :raw
+
+      file_options  = @options[:file_options]
+      parse_options = @options[:parse_options]
+
+      begin
+        csv = CSVWrapper.parse(@content, parse_options)
+        csv = csv.drop(1)  if file_options[:has_headers] == true  # drop the first row if it is a header 
+      rescue => e
+        Logger.new(@options).print(@data.path, :parse, e)
+      end
+
+      @state = :parsed
+        Data.new(@path, csv, @state, @options)
+    end
+
+
+
+    def convert_fields database, table_name
+      converter = Converter.new(self, database, table_name)
+      converter.convert
+    end
+
+
+    def into_db database, table_name
+      Storage.new(self, database, table_name).insert
+    end
+
+
+    def state= state
+      error_message = "#{state.inspect} is not a valid state. The following states are valid: #{print_valid_states}" 
+      raise ArgumentError, error_message  unless KnownStates.include?(state)
+      @state = state
+    end
+
+    # Helper methods --------------------------------
+
+    def print_valid_states
+      KnownStates.map { |s| s.inspect }.join(', ')
+    end
+
+  end
+end 
+ 

data/lib/rstore/exceptions.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+module RStore
+  # The error thrown when the length of a row does not fit the number of columns in the db table.
+  class InvalidRowLengthError < StandardError; end
+  class NullNotAllowedError   < StandardError; end
+  class InvalidStateError     < StandardError; end
+  class FileProcessingError   < StandardError; end
+
+end
+

data/lib/rstore/file_crawler.rb

@@ -0,0 +1,135 @@
+# encoding: utf-8
+
+require 'open-uri'
+require 'rstore/configuration'
+require 'rstore/data'
+require 'rstore/core_ext/string'
+
+module RStore
+  class FileCrawler
+
+    #attr_reader :file_options_hash
+    attr_reader :data_hash
+
+    attr_reader :file_options, :parse_options 
+    attr_reader :path
+    attr_reader :file_paths, :file_type
+    attr_reader :config
+
+
+
+    def initialize file_or_folder, file_type, options={}
+      @path                  = file_or_folder
+      @file_type             = file_type
+      @config                = Configuration.new(file_or_folder, options)
+      @file_options          = @config.file_options
+      @parse_options         = @config.parse_options
+      self.file_paths        = @path
+      self.file_options_hash = @file_paths
+      self.data_hash         = @file_options_hash
+    end
+
+
+    def file_paths= path
+      return @file_paths unless @file_paths.nil?
+
+      @file_paths = []
+      files       = []
+      if path.url?
+        return @file_paths << verify_and_format_url(path)
+      elsif File.directory?(File.expand_path(path))   # Directory
+        Dir.chdir(path) do                            # Change current directory to 'path'.
+          parse_directory(@file_options[:recursive]).each do |f|
+            files << File.expand_path(f)
+          end
+        end
+      else                                            # Either a file or a non-existing directory path
+        file = File.expand_path(path)
+        raise ArgumentError, "'#{path}' is not a valid path" unless File.exists?(file)
+
+        error_message = <<-MESSAGE.gsub(/^\s+/,'')
+                           Not a #{@file_type} file.
+                           NOTE: Non-#{@file_type} files in a directory path
+                                 are silently skipped WITHOUT raising an exception
+                           MESSAGE
+
+        raise ArgumentError, error_message                    unless can_read?(path)
+
+        files << file
+      end
+
+      @file_paths = files
+    rescue Exception => e
+        # Dirty hack to be able to call instantiate Logger.
+        data = Data.new(path, '', :raw, Configuration.default_options)
+
+        logger = Logger.new(data)
+        logger.log(:fetch, e)
+        logger.error
+    end
+
+
+    def data_hash= options_hash
+      hash = Hash[options_hash.map do |path, options|
+        data = Data.new(path, '', :raw, options)
+        [path, data]
+      end]
+      @data_hash = hash
+    end
+
+
+    def file_options_hash= file_paths
+      @file_options_hash unless @file_options_hash.nil?
+
+      hash = Hash.new {|h,k| h[k] = Hash.new {|h,k| h[k] = nil}}
+      file_paths.each do |path|
+        hash[path][:file_options]  = @file_options
+        hash[path][:parse_options] = @parse_options
+      end
+      @file_options_hash = hash
+    end
+
+
+    def parse_directory option
+      files = []
+      if option
+        files = Dir.glob("**/*.{#{@file_type}}") # Recursively read files into array, skip files that are not of @file_type
+      else
+        files = Dir.glob("*.{#{@file_type}}")    # Read files of the current directory
+      end
+      files.each do |file|
+        next if File.directory? file
+        file
+      end
+    end
+
+
+    # Helper methods ---------------------------
+
+
+    def can_read? path
+      !!(/.*\.#{@file_type.to_s}$/ =~ path)
+    end
+
+
+    def verify_and_format_url url
+      address = url
+      begin # add additional 'begin' block so that we can return the original, unchanged url in the error message.
+        open(address)
+        address
+      rescue
+        case address
+        when /^www/  # open-uri does not recognize URLs starting with 'www'
+          address = 'http://' + address
+          retry
+        when /^http:/ # open-uri does not redirect from http to https on a valid https URL 
+          address = address.gsub(/http/,'https')
+          retry
+        else 
+          raise ArgumentError, "Could not connect to #{url}. Please check if this URL is correct."
+        end
+      end
+    end
+
+  end
+end

data/lib/rstore/logger.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+
+require 'rstore/exceptions'
+
+module RStore
+  class Logger
+
+    attr_accessor :data
+    attr_accessor :message
+    
+
+    KnownStates = 
+      {:fetch   => "loading files", 
+       :parse   => "parsing file content",
+       :convert => "converting field values into their corresponding datatypes",
+       :store   => "storing file content into database"}
+
+
+
+    def initialize data_object
+      @data    = data_object
+      @message = ''
+    end
+
+
+    def log state, error, loc={}
+      raise ArgumentError "#{state} is an invalid state vor #{self.class}"  unless valid_state? state
+
+      loc = correct_location(loc)
+
+      type_of_error = error.class
+      error_message = error.to_s
+      location      = "Location     : #{location_to_s(loc)}" 
+      location      = loc.empty? ? '' : location
+
+      report = <<-TEXT.gsub(/^\s+/, '')
+      An error occured while #{KnownStates[state]}:
+      File         : #{@data.path} 
+      Type of error: #{type_of_error} 
+      Error message: #{error_message}
+      #{location}
+      =============
+      Please fix the error and run again.
+      NOTE: No data has been inserted into the database yet.
+      =============
+      TEXT
+
+      @message = report
+    end
+
+
+    def error
+      raise FileProcessingError, @message
+    end
+
+
+    # Helper methods ------------------------
+
+    def location_to_s location
+      location.map { |loc,val| "#{loc} #{val}" }.join(', ')
+    end
+
+
+    
+    def correct_location location
+ 
+      if location[:row]        # row_index
+        row = correct_row(location[:row])
+        if location[:col]      # col_index
+          col = location[:col]+1
+          {row: row, col: col}
+        else
+          {row: row}
+        end
+      else
+        location
+      end
+    end
+
+
+    def correct_row row
+      # row = row_index, which starts at 0
+      # Without headers: add 1 to row
+      # With headers   : add another 1 to row as the header row had been already removed
+      row = with_headers? ? row+2 : row+1
+      row
+    end
+
+
+    def valid_state? state
+      KnownStates.keys.any? { |val| val == state } 
+    end
+
+
+    def with_headers?
+      @data.options[:has_headers]
+    end
+
+  end
+end
+
+
+
+

data/lib/rstore/modules/helper_methods.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+module RStore
+  module HelperMethods
+
+    # Calulate primary key from schema
+    def p_key schema
+      schema.map do |(col_name, col_properties)|
+        col_name  if col_properties[:primary_key] == true
+      end.compact.first
+    end
+
+  end
+end

data/lib/rstore/storage.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+
+require 'sequel'
+require 'rstore/data'
+require 'rstore/logger'
+require 'rstore/exceptions'
+require 'rstore/modules/helper_methods'
+
+module RStore
+  class Storage
+    include HelperMethods
+
+    attr_accessor :data, :db, :table, :prepared_data, :primary_key
+    attr_accessor :state
+    
+
+    def initialize data_object, database, table_name
+      state = data_object.state
+      raise InvalidStateError, "#{state.inspect} is not a valid state on initialization for class Storage" unless state == :converted
+      @state = state
+      @data  = data_object.clone
+      @db    = database
+      @table = table_name
+      @schema = @db.schema(@table)
+      @primary_key = p_key @schema 
+      @prepared_data = prepare_data
+    end
+
+    
+    def column_names
+      @schema.map do |(col_name, col_properties)|  
+        col_name unless col_name == @primary_key
+      end.compact
+    end
+
+
+    def prepare_data
+      col_names = column_names
+      @data.content.map do |row|
+        Hash[col_names.zip(row)]
+      end
+    end
+
+    
+    def insert
+      dataset = @db[@table]
+      begin
+        @db.transaction do
+          @prepared_data.each_with_index do |row, row_index|
+            @row_index = row_index
+            dataset.insert(row)
+            # Sequel often only throws an exception when retrieving an incorrect record, 
+            # The following therefore is to catch invalid data of data types that are
+            # not checked by RStore::Converter 
+            dataset.order(@primary_key).last
+          end
+        end
+      rescue Exception => e
+        logger = Logger.new(@data)
+        logger.log(:store, e, row: @row_index)
+        logger.error
+      
+      end
+      @state = :stored
+      @state
+    end
+  end
+end
+
+ 
+ 

data/lib/rstore/version.rb

@@ -0,0 +1,3 @@
+module RStore
+  VERSION = "0.2.0"
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: rstore
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- Stefan Rohlfing
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-27 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &17766760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *17766760
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &17766300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *17766300
+description: ! '+ Batch processing of csv files
+
+  + Fetches data from different sources: files, directories, URLs
+
+  + Customizable using additional options 
+
+  + Validation of field values. At the moment validation of the following types is
+  supported
+
+  + Descriptive error messages pointing helping you to find any invalid data quickly
+
+  + Safe and transparent data storage:
+
+  + -- Using database transactions: Either the data from all files is stored or none
+
+  + -- The data storage method can only be executed once for every instance of RStore::CSV
+
+'
+email: stefan.rohlfing@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/rstore/version.rb
+- lib/rstore/csv.rb
+- lib/rstore/configuration.rb
+- lib/rstore/converter.rb
+- lib/rstore/base_db.rb
+- lib/rstore/data.rb
+- lib/rstore/file_crawler.rb
+- lib/rstore/logger.rb
+- lib/rstore/storage.rb
+- lib/rstore/base_table.rb
+- lib/rstore/core_ext/hash.rb
+- lib/rstore/core_ext/csv_wrapper.rb
+- lib/rstore/core_ext/string.rb
+- lib/rstore/core_ext/object.rb
+- lib/rstore/exceptions.rb
+- lib/rstore/modules/helper_methods.rb
+- lib/rstore.rb
+- README.md
+- Rakefile
+- LICENSE
+homepage: http://github.com/bytesource/rstore
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: rstore
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: RStore - A library for easy batch storage of csv data into a database
+test_files: []