mdarray 0.4.3.pre-java → 0.5.0-java

data/test/colt/VALE3_short-err.csv

@@ -0,0 +1,20 @@
+Date,Open,High,Low,Close,Volume,Adj Close
+2013-04-24,33.58,34.37,33.42,33.58,7022800,33.58
+2013-04-23,32.53,33.54,32.07,33.40,3379500,33.40
+2013-04-22,32.14,32.56,31.80,32.39,2287100,32.39
+2013-04-19,32.34,32.54,31.54,32.10,4675200,32.10
+2013-04-18,32.45,32.78,31.72,31.90,4258300,31.90
+2013-04-17,31.70,32.50,31.07,32.46,9355000,32.46
+2013-04-16,33.05,33.31,32.69,32.33,5677000,32.33
+2013-04-15,33.55,33.86,32.05,32.46,4378700,32.46
+2013-04-12,34.12,35.86,33.34,34.58,6132200,34.58
+2013-04-11,35.15,35.29,33.89,34.14,5182600,34.14
+2013-04-10,36.43,36.81,34.85,35.00,8489500,35.00
+2013-04-09,35.00,36.60,35.00,36.19,7892300,36.19
+2013-04-08,34.69,34.88,34.08,34.70,6448600,34.70
+2013-04-05,33.74,34.83,33.44,34.56,4539600,34.56
+2013-04-04,34.95,35.20,34.09,34.19,4226900,34.19
+2013-04-03,33.18,35.35,33.06,35.16,6258100,35.16
+2013-04-02,34.81,34.92,33.18,33.25,5340900,33.25
+2013-04-01,34.81,34.92,33.94,34.00,1962600
+2013-03-29,35.13,35.13,35.13,35.13,000,35.13

data/test/colt/VALE3_short.csv

@@ -0,0 +1,20 @@
+Date,Open,High,Low,Close,Volume,Adj Close
+2013-04-24,33.58,34.37,33.42,33.58,7022800,33.58
+2013-04-23,32.53,33.54,32.07,33.40,3379500,33.40
+2013-04-22,32.14,32.56,31.80,32.39,2287100,32.39
+2013-04-19,32.34,32.54,31.54,32.10,4675200,32.10
+2013-04-18,32.45,32.78,31.72,31.90,4258300,31.90
+2013-04-17,31.70,32.50,31.07,32.46,9355000,32.46
+2013-04-16,33.05,33.31,32.69,32.33,5677000,32.33
+2013-04-15,33.55,33.86,32.05,32.46,4378700,32.46
+2013-04-12,34.12,35.86,33.34,34.58,6132200,34.58
+2013-04-11,35.15,35.29,33.89,34.14,5182600,34.14
+2013-04-10,36.43,36.81,34.85,35.00,8489500,35.00
+2013-04-09,35.00,36.60,35.00,36.19,7892300,36.19
+2013-04-08,34.69,34.88,34.08,34.70,6448600,34.70
+2013-04-05,33.74,34.83,33.44,34.56,4539600,34.56
+2013-04-04,34.95,35.20,34.09,34.19,4226900,34.19
+2013-04-03,33.18,35.35,33.06,35.16,6258100,35.16
+2013-04-02,34.81,34.92,33.18,33.25,5340900,33.25
+2013-04-01,34.81,34.92,33.94,34.00,1962600,34.00
+2013-03-29,35.13,35.13,35.13,35.13,000,35.13

data/test/colt/test_complete.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+##########################################################################################
+# Copyright © 2013 Rodrigo Botafogo. All Rights Reserved. Permission to use, copy, modify, 
+# and distribute this software and its documentation, without fee and without a signed 
+# licensing agreement, is hereby granted, provided that the above copyright notice, this 
+# paragraph and the following two paragraphs appear in all copies, modifications, and 
+# distributions.
+#
+# IN NO EVENT SHALL RODRIGO BOTAFOGO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, 
+# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF 
+# THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF RODRIGO BOTAFOGO HAS BEEN ADVISED OF THE 
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# RODRIGO BOTAFOGO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE 
+# SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". 
+# RODRIGO BOTAFOGO HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, 
+# OR MODIFICATIONS.
+##########################################################################################
+
+#=begin
+require_relative 'test_statistics'
+require_relative 'test_stat_list'
+#=end

data/test/colt/test_stat_list.rb

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+
+##########################################################################################
+# Copyright © 2013 Rodrigo Botafogo. All Rights Reserved. Permission to use, copy, modify, 
+# and distribute this software and its documentation, without fee and without a signed 
+# licensing agreement, is hereby granted, provided that the above copyright notice, this 
+# paragraph and the following two paragraphs appear in all copies, modifications, and 
+# distributions.
+#
+# IN NO EVENT SHALL RODRIGO BOTAFOGO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, 
+# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF 
+# THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF RODRIGO BOTAFOGO HAS BEEN ADVISED OF THE 
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# RODRIGO BOTAFOGO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE 
+# SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". 
+# RODRIGO BOTAFOGO HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, 
+# OR MODIFICATIONS.
+##########################################################################################
+
+require 'rubygems'
+require "test/unit"
+require 'shoulda'
+
+require 'mdarray'
+
+class MDArrayTest < Test::Unit::TestCase
+
+  context "Parrallel Colt Integration" do
+
+    setup do
+            
+    end # setup
+
+    #-------------------------------------------------------------------------------------
+    #
+    #-------------------------------------------------------------------------------------
+
+    should "work with stat_lists" do
+
+      # Creates a DoubleStatList of capacity 10
+      @list = DoubleStatList.new(10)
+
+      @list.add(2.25)
+      @list.add(5.0)
+
+      assert_equal(2.25, @list[0])
+      assert_equal(5.0, @list[1])
+      assert_equal(3.625, @list.mean)
+
+      # Search for element in the list
+      assert_equal(1, @list.binary_search(5))
+      assert_equal(1, @list.index_of(5.0))
+
+      @list.add(3.0)
+      @list.add(5.0)
+      
+      assert_equal(3, @list.last_index_of(5.0))
+
+      list2 = @list.copy
+      # changing the value of @list
+      @list[0] = 10
+      assert_equal(10, @list[0])
+      # list2 is unchanged
+      assert_equal(2.25, list2[0])
+
+      @list.shuffle
+      @list.print
+
+      # mean is now wrong... we've added/changed elements in @list, but we still get the
+      assert_equal(3.625, @list.mean)
+
+      # to make it right we need to reset_statistics.  Whenever an element on the 
+      # list is changed it is required to call reset_statistics
+      @list.reset_statistics
+      assert_equal(5.75, @list.mean)
+
+    end
+    
+    #-------------------------------------------------------------------------------------
+    #
+    #-------------------------------------------------------------------------------------
+
+    should "be memory efficient when possible" do
+
+      # read file VALE3.  This file has a header that we need to discard.  VALE3 
+      # contains quote values from brazilian company VALE as obtained from Yahoo finance
+      # (quote vale3.SA)
+      vale3 = MDArray.double("#{$COLT_TEST_DIR}/VALE3_short.csv", true)
+
+      # calling reset_statistics on vale3, creates a stat_list for vale3
+      vale3.reset_statistics
+
+      # get vale3 stat_list.  This should not copy any data... let's check it
+      list = vale3.stat_list
+      
+      # let's change a value in list
+      list[0] = 10
+      assert_equal(10, list[0])
+      assert_equal(10, vale3[0, 0])
+
+      # Getting the open value
+      open = vale3.slice(1, 1)
+      
+      # open and vale3 should use the same backing store
+      open[0] = 5
+      assert_equal(5, open[0])
+      assert_equal(5, vale3[0,1])
+
+      # now... doing reset_statistics on open will not use the same backing store
+      # as open is not contigous in memory and copying is necessary.  This is the
+      # cost integrating NetCDF and Parallel Colt.  It might actually be a good 
+      # idea in some cases to make a copy and operate on contigous memory than using
+      # indexing to move over the array.
+      open.reset_statistics
+      open_list = open.stat_list
+      open_list[0] = 1000
+      assert_equal(1000, open_list[0])
+      assert_equal(5, open[0])
+
+      
+
+    end
+    
+  end
+  
+end

data/test/colt/test_statistics.rb

@@ -0,0 +1,114 @@
+require 'rubygems'
+require "test/unit"
+require 'shoulda'
+
+require 'mdarray'
+
+class MDArrayTest < Test::Unit::TestCase
+
+  context "Statistics Tests" do
+
+    setup do
+
+    end
+
+    #-------------------------------------------------------------------------------------
+    #
+    #-------------------------------------------------------------------------------------
+
+    should "do stats operations" do
+
+      # read file VALE3.  This file has a header that we need to discard.  VALE3 
+      # contains quote values from brazilian company VALE as obtained from Yahoo finance
+      # (quote vale3.SA)
+      vale3 = MDArray.double("#{$COLT_TEST_DIR}/VALE3_short.csv", true)
+
+      # in order to use statistics we need to reset_statistics on array vale3.  This 
+      # breaks version 0.4.3 statistics that did not require a call to reset_statistics.
+      # In future version we will keep both methods with reset_statistics and without
+      # reset_statistics being required.
+      vale3.reset_statistics
+
+      # sum all values of vale3.  This does not make sense from a financial point of view.
+      # suming all values including dates... Not doing anything with the value, just
+      # checking that it does not crash.
+      vale3.sum
+
+      # lets get only the open price for the whole period. We slice the vale3 on the
+      # second dimension and get the second column. 
+      open = vale3.slice(1,1)
+      
+      # lets also get the high value.
+      high = vale3.slice(1,2)
+
+      # weights to be used for weighted operations
+      weights = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14.0, 15, 16, 17, 18, 19]
+
+      # splitters to be used to split the list
+      splitters = [34.0, 36.0]
+
+      # quantiles to be used to split the list
+      percs = [0.20, 0.40, 0.60, 0.80, 1]
+
+      # getting descriptive statistics for the open value.  open is a new MDArray, so
+      # we need to reset_statistics for open as well.
+      open.reset_statistics
+
+      assert_equal(-0.30204751376121775, open.auto_correlation(10))
+      assert_equal(0.8854362245369992, open.correlation(high))
+      assert_equal(1.4367963988919659, open.covariance(high))
+      assert_equal(0.00079607686762408, open.durbin_watson)
+      assert_equal(33.837262944797345, open.geometric_mean)
+      assert_equal(33.81400448777291, open.harmonic_mean)
+      assert_equal(-0.925644222523478, open.kurtosis)
+      assert_equal(0.681656774667894, open.lag1)
+      assert_equal(36.43, open.max)
+      assert_equal(33.86052631578948,open.mean)
+      assert_equal(1.0889750692520779,open.mean_deviation)
+      assert_equal(33.74,open.median)
+      assert_equal(31.7,open.min)
+      assert_equal(0.07736522466830013,open.moment3)
+      assert_equal(5.147382269264963,open.moment4)
+      assert_equal(34.17368421052632,open.pooled_mean(high))
+      assert_equal(1.623413296398882,open.pooled_variance(high))
+      assert_equal(1.1442193777839571e+29,open.product)
+      assert_equal(32.498000000000005,open.quantile(0.2))
+      assert_equal(0.8421052631578947,open.quantile_inverse(35.0))
+      assert_equal(5.903846153846159,open.rank_interpolated(33.0))
+      assert_equal(33.88377930514836,open.rms)
+      assert_equal(-0.8280585298104861,open.sample_kurtosis)
+      assert_equal(1.5166184210526306,open.sample_covariance(high))
+      assert_equal(1.0142698435367294,open.sample_kurtosis_standard_error)
+      assert_equal(0.042567930807996486,open.sample_skew)
+      assert_equal(0.5237666950104207,open.sample_skew_standard_error)
+      assert_equal(1.3075102994156926,open.sample_standard_deviation)
+      assert_equal(1.6627719298244807,open.sample_variance)
+      assert_equal(1.2035654385963137,open.sample_weighted_variance(weights))
+      assert_equal(0.039130771304858564,open.skew)
+      assert_equal(1.255092672964214,open.standard_deviation)
+      assert_equal(0.28793800664365016,open.standard_error)
+      assert_equal(643.35,open.sum)
+      assert_equal(0.561897364355954,open.sum_of_inversions)
+      assert_equal(66.90969033519778,open.sum_of_logarithms)
+      assert_equal(29.92989473684211,open.sum_of_power_deviations(2, open.mean))
+      assert_equal(740665.2440910001,open.sum_of_powers(3))
+      assert_equal(21814.099500000004,open.sum_of_squares)
+      assert_equal(28.354637119112198,open.sum_of_squared_deviations)
+      assert_equal(33.862, open.trimmed_mean(2, 2))
+      assert_equal(1.5752576177284554,open.variance)
+      assert_equal(34.31689473684211,open.weighted_mean(weights))
+      assert_equal(0.029110571117388302,open.weighted_rms(weights))
+      assert_equal(33.816315789473684,open.winsorized_mean(1, 1))
+
+      p "Distinct values: #{open.frequencies[:distinct_values]}"
+      p "Frequencies: #{open.frequencies[:frequencies]}"
+      p "Split: #{open.split(splitters)}"
+      p "Quantiles: #{open.quantiles(percs)}"
+      p "Sorted elements: #{open.sort}"
+      p "Standardized elements: #{open.standardize}"
+
+    end
+
+  end
+
+end

data/test/complete.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+
+##########################################################################################
+# Copyright © 2013 Rodrigo Botafogo. All Rights Reserved. Permission to use, copy, modify, 
+# and distribute this software and its documentation, without fee and without a signed 
+# licensing agreement, is hereby granted, provided that the above copyright notice, this 
+# paragraph and the following two paragraphs appear in all copies, modifications, and 
+# distributions.
+#
+# IN NO EVENT SHALL RODRIGO BOTAFOGO BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, 
+# INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF 
+# THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF RODRIGO BOTAFOGO HAS BEEN ADVISED OF THE 
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# RODRIGO BOTAFOGO SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE 
+# SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED HEREUNDER IS PROVIDED "AS IS". 
+# RODRIGO BOTAFOGO HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, 
+# OR MODIFICATIONS.
+##########################################################################################
+
+require 'env.rb'
+
+require 'mdarray/test_complete'
+require 'colt/test_complete'

data/test/env.rb

@@ -1,50 +1,74 @@
 require 'rbconfig'
 
-# Config::CONFIG['host_os'] # returns mswin32 on Windows, for example
-# p Config::CONFIG
+# Home directory for MDArray
+$MDARRAY_HOME_DIR = ".."
 
-# Fix environment variable for MultiCell
-# Prepare environment to work inside Cygwin
+# MDArray Test directory
+$MDARRAY_TEST_DIR = "./mdarray"
 
-# Return the cygpath of a path
-def cygpath(path)
-  `cygpath -p -m #{path}`.tr("\n", "")
-end
+# Colt Test directory
+$COLT_TEST_DIR = "./colt"
 
-# Add path to load path
-def mklib(path, home_path = true)
+##########################################################################################
+# If we need to test for coverage
+##########################################################################################
 
-  if (home_path)
-    lib = path + "/lib"
-  else
-    lib = path
-  end
+if ENV['COVERAGE'] == 'true'
 
-  $LOAD_PATH << `cygpath -p -m #{lib}`.tr("\n", "")
+  require 'simplecov'
+  
+  SimpleCov.start do
+    @filters = []
+    add_group "MDArray", "lib/mdarray"
+    add_group "Colt", "lib/colt"
+  end
 
 end
 
-# Home directory for MDArray
-$MDARRAY_HOME_DIR = "/home/zxb3/Desenv/MDArray"
-mklib($MDARRAY_HOME_DIR)
+##########################################################################################
+# Prepare environment to work inside Cygwin
+##########################################################################################
 
-# Home directory for MultiCell
-$MC_HOME_DIR = "/home/zxb3/Desenv/MultiCell"
-mklib($MC_HOME_DIR)
+if ENV['MDARRAY_ENV'] == 'cygwin'
 
-# Home directory for NcInterface
-$NCI_HOME_DIR = "/home/zxb3/Desenv/NcInterface"
-mklib($NCI_HOME_DIR)
+  # RbConfig::CONFIG['host_os'] # returns mswin32 on Windows, for example
+  # p Config::CONFIG
+  
+  #---------------------------------------------------------------------------------------
+  # Return the cygpath of a path
+  #---------------------------------------------------------------------------------------
 
-# Home directory for BM&F Bovespa files
-$BMF_BOVESPA_DIR = "/home/zxb3/Investimentos/SeriesHistoricas/BMF_BOVESPA/"
+  def cygpath(path)
+    `cygpath -a -p -m #{path}`.tr("\n", "")
+  end
+
+  #---------------------------------------------------------------------------------------
+  # Add path to load path
+  #---------------------------------------------------------------------------------------
+
+  def mklib(path, home_path = true)
+    
+    if (home_path)
+      lib = path + "/lib"
+    else
+      lib = path
+    end
+    
+    $LOAD_PATH << `cygpath -p -m #{lib}`.tr("\n", "")
+    
+  end
+      
+  mklib($MDARRAY_HOME_DIR)
 
-# Home directory for results
-$RESULT_HOME_DIR = $MC_HOME_DIR + "/results/"
+  $MDARRAY_TEST_DIR = cygpath($MDARRAY_TEST_DIR)
+  $COLT_TEST_DIR = cygpath($COLT_TEST_DIR)
+  
+end
 
 =begin
 # Build Jruby classpath from environment classpath
 ENV['WCLASSPATH'].split(';').each do |path|
-  $CLASSPATH << cygpath(path)
+$CLASSPATH << cygpath(path)
 end
 =end
+