mdarray-jcsv 0.6.3-java

data/lib/map_reader.rb

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+
+##########################################################################################
+# author Rodrigo Botafogo
+#
+# Copyright © 2015 Rodrigo Botafogo. All Rights Reserved. Permission to use, copy, modify, 
+# and distribute this software and its documentation, without fee and without a signed 
+# licensing agreement, is hereby granted, provided that the above copyright notice, this 
+# paragraph and the following two paragraphs appear in all copies, modifications, and 
+# distributions.
+#
+# IN NO EVENT SHALL RODRIGO BOTAFOGO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, 
+# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF 
+# THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF RODRIGO BOTAFOGO HAS BEEN ADVISED OF THE 
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# RODRIGO BOTAFOGO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE 
+# SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". 
+# RODRIGO BOTAFOGO HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, 
+# OR MODIFICATIONS.
+##########################################################################################
+
+require 'critbit'
+
+class Jcsv
+
+  #========================================================================================
+  #
+  #========================================================================================
+
+  class MapReader < Reader
+    include_package "java.io"
+
+    #---------------------------------------------------------------------------------------
+    #
+    #---------------------------------------------------------------------------------------
+
+    def initialize(*params)
+      super(*params)
+      @column_mapping.mapping = @headers if !@dimensions
+      @map_klass = (@format == :map)? Hash : Critbit
+    end
+    
+    #---------------------------------------------------------------------------------------
+    # Maps columns to the given names.  In map reader, there is no column reordering, as
+    # this does not really make any sense, since one gets to the data through the key and
+    # not through its position in the array.  If there are dimensions set, then every
+    # dimension will map to true, in order for it to be properly processed by the parsing
+    # method. Other fields can still be mapped to false, so that they are not read if
+    # desired.
+    #---------------------------------------------------------------------------------------
+
+    def mapping=(column_mapping)
+
+      @column_mapping.mapping ||= Array.new
+      
+      @headers.each_with_index do |h, i|
+        next if @dimensions && !@dimensions[h].nil?
+        name = column_mapping[h]
+        raise ArgumentError.new("'true' is not allowed as a mapping: #{column_mapping}") if
+          name == true
+        @column_mapping.mapping[i] = (name.nil?)? h : name
+      end
+
+    end
+    
+    #---------------------------------------------------------------------------------------
+    # read the file.
+    #---------------------------------------------------------------------------------------
+    
+    def read(&block)
+
+      # When no block given, chunks read are stored in an array and returned to the user.
+      if (!block_given?)
+        # if dimensions and chunk_size is 0, then do not wrap each row in an array, we
+        # can access the data directly by using the dimension key
+        if (@dimensions && @chunk_size == 0)
+          rows = @map_klass.new
+          parse_with_block do |line_no, row_no, chunk|
+            rows.merge!(chunk)
+          end
+        else
+          # chunk_size > 0, then each chunk should be a hash, and all chunks should
+          # be wrapped inside an array
+          rows = []
+          parse_with_block do |line_no, row_no, chunk|
+            rows << chunk
+          end
+        end
+        rows
+      else # block given
+        parse_with_block(&block)
+      end
+    end
+    
+    #---------------------------------------------------------------------------------------
+    #
+    #---------------------------------------------------------------------------------------
+
+    private
+    
+    #---------------------------------------------------------------------------------------
+    #
+    #---------------------------------------------------------------------------------------
+
+    def new_reader(preferences)
+      
+      begin
+        raise MissingHeadersError.new("Reading file as map requires headers.") if
+          (!@headers && !@custom_headers)
+        @reader = CMR.new(FileReader.new(@filename), preferences, @dimensions,
+                          @suppress_warnings)
+      rescue java.io.IOException => e
+        raise IOError.new(e.message)
+      end
+
+    end
+
+    #---------------------------------------------------------------------------------------
+    #
+    #---------------------------------------------------------------------------------------
+
+    def format(chunk)
+      chunk
+    end
+    
+    #---------------------------------------------------------------------------------------
+    # Maps columns to the given names.  In map reader, there is no column reordering, as
+    # this does not really make any sense, since one gets to the data through the key and
+    # not through its position in the array.  If there are dimensions set, then every
+    # dimension will map to true, in order for it to be properly processed by the parsing
+    # method. Other fields can still be mapped to false, so that they are not read if
+    # desired.
+    #---------------------------------------------------------------------------------------
+
+    def assign_mapping(column_mapping)
+
+      @column_mapping.mapping ||= Array.new
+      
+      @headers.each_with_index do |h, i|
+        name = column_mapping[h]
+        @column_mapping.mapping[i] = (name.nil?)? h : name
+      end
+
+    end
+    
+    #---------------------------------------------------------------------------------------
+    # A chunk is either one row of the file, or an array with rows.  One row can be either
+    # a one dimensional array with all columns or a hash with all columns (excluding the
+    # dimensions).
+    #---------------------------------------------------------------------------------------
+
+    def read_chunk
+
+      if (@dimensions)
+        if (@chunk_size == 0)
+          row = @reader.read(@column_mapping, @filters)
+          return (row.nil?)? nil : { row.delete(:key).join(".") => row }
+        end
+
+        rows = {}
+        (1..@chunk_size).each do |i|
+          if ((row = @reader.read(@column_mapping, @filters)).nil?)
+            return (rows.size == 0)? nil : rows
+          else
+            if (@deep_map)
+              key = row.delete(:key)
+              key.reduce(rows) { |h,m| h[m] ||= {} }
+              last = key.pop
+              if (key.inject(rows, :fetch)[last] != {})
+                # p "overriding value for key: #{chunk[:key]} with #{chunk}"
+                raise DuplicateKeyError.new("Key #{row[:key]} not unique for this dataset. #{row}")
+              end
+              key.inject(rows, :fetch)[last] = row
+            else # not a deep map
+              key = row.delete(:key).join(".")
+              raise DuplicateKeyError.new("Key #{key} not unique for this dataset. #{row}") if
+                rows.has_key?(key)
+              rows.merge!({key => row})
+            end
+          end
+        end
+        return rows
+      else # no dimensions
+        super
+      end
+    end
+      
+  end
+    
+end

data/lib/mdarray-jcsv.rb

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+##########################################################################################
+# @author Rodrigo Botafogo
+#
+# Copyright © 2015 Rodrigo Botafogo. All Rights Reserved. Permission to use, copy, modify, 
+# and distribute this software and its documentation, without fee and without a signed 
+# licensing agreement, is hereby granted, provided that the above copyright notice, this 
+# paragraph and the following two paragraphs appear in all copies, modifications, and 
+# distributions.
+#
+# IN NO EVENT SHALL RODRIGO BOTAFOGO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, 
+# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF 
+# THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF RODRIGO BOTAFOGO HAS BEEN ADVISED OF THE 
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# RODRIGO BOTAFOGO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE 
+# SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". 
+# RODRIGO BOTAFOGO HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, 
+# OR MODIFICATIONS.
+##########################################################################################
+
+require_relative 'jcsv'

data/lib/mdarray_reader.rb

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+##########################################################################################
+# @author Rodrigo Botafogo
+#
+# Copyright © 2015 Rodrigo Botafogo. All Rights Reserved. Permission to use, copy, modify, 
+# and distribute this software and its documentation, without fee and without a signed 
+# licensing agreement, is hereby granted, provided that the above copyright notice, this 
+# paragraph and the following two paragraphs appear in all copies, modifications, and 
+# distributions.
+#
+# IN NO EVENT SHALL RODRIGO BOTAFOGO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, 
+# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF 
+# THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF RODRIGO BOTAFOGO HAS BEEN ADVISED OF THE 
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# RODRIGO BOTAFOGO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE 
+# SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". 
+# RODRIGO BOTAFOGO HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, 
+# OR MODIFICATIONS.
+##########################################################################################
+
+require 'mdarray'
+
+class Jcsv
+
+  #========================================================================================
+  #
+  #========================================================================================
+  
+  class MDArrayReader < MapReader
+    include_package "java.io"
+
+    #---------------------------------------------------------------------------------------
+    #
+    #---------------------------------------------------------------------------------------
+
+    def initialize(*params)
+
+      filter = nil
+      
+      @dtype = params[1].delete(:dtype)
+      
+      case @dtype
+      when :byte, :short, :int
+        filter = Jcsv.int
+      when :long
+        filter = Jcsv.long
+      when :float, :double
+        filter = Jcsv.double
+      else
+        raise "Cannot create MDArray of dtype '#{@dtype}'"
+      end
+      
+      params[1][:default_filter] = filter
+      super(*params)
+      
+    end
+
+    #---------------------------------------------------------------------------------------
+    #
+    #---------------------------------------------------------------------------------------
+
+    def read
+      to_mdarray(@dtype, super)
+    end
+
+    #---------------------------------------------------------------------------------------
+    # Converts the data to an MDArray
+    #---------------------------------------------------------------------------------------
+
+    def to_mdarray(dtype, storage)
+
+      raise "Cannot convert deep map into MDArray" if (@deep_map == true)
+      
+      prod = nil
+      shape = []
+      vector = []
+      
+      columns = @column_mapping.mapping - [true, false, nil]
+      header_size = columns.size
+
+      if (@dimensions.nil?)
+        shape = [storage.size, header_size]
+        storage.each do |line|
+          vector.concat(line.values)
+        end
+      else
+        @dimensions.dimensions_names.each do |name|
+          keys = @dimensions[name].labels.keys
+          shape << keys.size
+          prod = (prod.nil?)? keys : prod.product(keys)
+        end
+        
+        shape << header_size
+        
+        prod.each do |k|
+          row = (@dimensions.dimensions_names.size > 1)? storage[k.flatten.join(".")] : storage[k]
+          vector.concat(((row.nil?)? ([Float::NAN] * header_size) : row.values))
+        end
+      end
+
+      array = MDArray.build(@dtype, shape, vector)
+      
+    end
+    
+  end
+  
+end

data/lib/numeric_filters.rb

@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+
+##########################################################################################
+# @author Rodrigo Botafogo
+#
+# Copyright © 2015 Rodrigo Botafogo. All Rights Reserved. Permission to use, copy, modify, 
+# and distribute this software and its documentation, without fee and without a signed 
+# licensing agreement, is hereby granted, provided that the above copyright notice, this 
+# paragraph and the following two paragraphs appear in all copies, modifications, and 
+# distributions.
+#
+# IN NO EVENT SHALL RODRIGO BOTAFOGO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, 
+# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF 
+# THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF RODRIGO BOTAFOGO HAS BEEN ADVISED OF THE 
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# RODRIGO BOTAFOGO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE 
+# SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". 
+# RODRIGO BOTAFOGO HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, 
+# OR MODIFICATIONS.
+##########################################################################################
+
+require 'bigdecimal'
+require_relative 'locale'
+
+class Jcsv
+  # include_package "org.supercsv.cellprocessor"
+  # include_package "org.supercsv.cellprocessor.constraint"
+
+  #========================================================================================
+  #
+  #========================================================================================
+  
+  class RBParseInt < org.supercsv.cellprocessor.ParseInt
+    include NextFilter
+
+    def execute(value, context)
+      begin
+        exec_next(super(value, context), context)
+      rescue org.supercsv.exception.SuperCsvCellProcessorException => e
+        raise FilterError.new("#{e.message} in #{context}")
+      end
+    end
+    
+  end
+
+  #========================================================================================
+  #
+  #========================================================================================
+  
+  class RBParseLong < org.supercsv.cellprocessor.ParseLong
+    include NextFilter
+
+    def execute(value, context)
+      begin
+        exec_next(super(value, context), context)
+      rescue org.supercsv.exception.SuperCsvCellProcessorException => e
+        raise FilterError, "#{e.message} in #{context}"
+      end
+    end
+    
+  end
+
+  #========================================================================================
+  #
+  #========================================================================================
+  
+  class RBParseDouble < org.supercsv.cellprocessor.ParseDouble
+    include NextFilter
+
+    def execute(value, context)
+      begin
+        exec_next(super(value, context), context)
+      rescue org.supercsv.exception.SuperCsvCellProcessorException => e
+        raise FilterError, "#{e.message} in #{context}"
+      end
+    end
+    
+  end
+
+  #========================================================================================
+  #
+  #========================================================================================
+
+  class RBParseFloat < Filter
+    
+    attr_reader :locale
+    attr_reader :dfs
+    
+    def initialize(locale)
+      @locale = locale
+      @dfs = DFSymbols.new(locale)
+      @grouping_separator = @dfs.grouping_separator
+      @decimal_separator = @dfs.decimal_separator
+      super()
+    end
+    
+    def execute(value, context)
+      validateInputNotNull(value, context)
+      value = value.gsub(@grouping_separator.chr, "").
+              gsub(@decimal_separator.chr, ".").to_f
+      exec_next(value, context)
+    end
+    
+  end
+
+  #========================================================================================
+  #
+  #========================================================================================
+  
+  class RBParseBignum < Filter
+        
+    def execute(value, context)
+      validateInputNotNull(value, context)
+      exec_next(value.to_i, context)
+    end
+
+  end
+
+  #========================================================================================
+  #
+  #========================================================================================
+  
+  class RBParseComplex < Filter
+        
+    def execute(value, context)
+      validateInputNotNull(value, context)
+      exec_next(value.to_c, context)
+    end
+
+  end
+
+  #========================================================================================
+  #
+  #========================================================================================
+  
+  class RBParseRational < Filter
+        
+    def execute(value, context)
+      validateInputNotNull(value, context)
+      exec_next(value.to_r, context)
+    end
+
+  end
+
+  #========================================================================================
+  #
+  #========================================================================================
+
+  class RBParseBigDecimal < Filter
+    
+    attr_reader :locale
+    attr_reader :dfs
+    
+    def initialize(locale)
+      @locale = locale
+      @dfs = DFSymbols.new(locale)
+      @grouping_separator = @dfs.grouping_separator
+      @decimal_separator = @dfs.decimal_separator
+      super()
+    end
+    
+    def execute(value, context)
+      validateInputNotNull(value, context)
+      # raise "BigDecimal expects a String as input not #{value}" if !(value.is_a? String)
+      bd = BigDecimal.new(value.gsub(@grouping_separator.chr, "").
+                           gsub(@decimal_separator.chr, "."))
+      exec_next(bd, context)
+    end
+    
+  end
+
+  #========================================================================================
+  #
+  #========================================================================================
+
+  def self.int
+    RBParseInt.new
+  end
+
+  def self.long
+    RBParseLong.new
+  end
+
+  def self.double
+    RBParseDouble.new
+  end
+
+  def self.fixnum
+    RBParseBignum.new
+  end
+
+  def self.float(locale = Locale.default)
+    RBParseFloat.new(locale)
+  end
+
+  def self.complex
+    RBParseComplex.new
+  end
+
+  def self.rational
+    RBParseRational.new
+  end
+  
+  def self.bignum
+    RBParseBignum.new
+  end
+
+  #---------------------------------------------------------------------------------------
+  # Convert a String to a BigDecimal. It uses the String constructor of BigDecimal
+  # (new BigDecimal("0.1")) as it yields predictable results (see BigDecimal).
+  # If the data uses a character other than "." as a decimal separator (Germany uses ","
+  # for example), then use the constructor that accepts a DecimalFormatSymbols object, as
+  # it will convert the character to a "." before creating the BigDecimal. Likewise if the
+  # data contains a grouping separator (Germany uses "." for example) then supplying a
+  # DecimalFormatSymbols object will allow grouping separators to be removed before
+  # parsing.
+  #---------------------------------------------------------------------------------------
+  
+  def self.big_decimal(locale = Locale.default)
+    Jcsv::RBParseBigDecimal.new(locale)
+  end
+  
+end