extcsv 0.12.1 → 0.12.2

data/gemspec

@@ -2,7 +2,7 @@ require 'rubygems'
 
 spec = Gem::Specification.new do |s|
   s.name              = "extcsv"
-  s.version           = "0.12.1"
+  s.version           = "0.12.2"
   s.date              = Time.new.strftime("%Y-%m-%d")
   s.author            = "Ralf Mueller"
   s.email             = "stark.dreamdetective@gmail.com"

data/lib/extcsv.rb

@@ -30,7 +30,7 @@ end
 # ==== License: BSD - see {license file}[http:/extcsv.rubyforge.org/svn/extcsv/trunk/LICENSE]
 ################################################################################
 class ExtCsv < OpenStruct
-  VERSION = '0.12.0'
+  VERSION = '0.12.2'
 
   include Comparable
   include Enumerable
@@ -48,6 +48,9 @@ class ExtCsv < OpenStruct
   # Non-Data fields
   METADATA = %w{mode datatype datacolumns cellsep rowsep filename filemtime}
 
+  # ShunkSize for handling large objects with MRI
+  ShunkSize = 65536
+
   # mode can be one of the allowed MODES
   # datatype can be one of the TYPES
   #
@@ -133,7 +136,7 @@ class ExtCsv < OpenStruct
     # remove blank lines
     filecontent = filecontent.gsub(/\r\r/,"\r").gsub(/(\r\n){2,}/,"\r\n").gsub(/\n{2,}/,"\n")
     csv         = CSV.parse(filecontent, :col_sep => obj_hash[:cellsep])#, obj_hash[:rowsep])
-    
+
     # read @datatype specific header
     header = csv.shift
     # remove comments sign from the header
@@ -252,7 +255,7 @@ class ExtCsv < OpenStruct
       end
     }
     # default is the lookup in the whole array of values for each var
-    lookup = (0..@table[vars[0]].size-1).to_a
+    lookup = (0...@table[vars[0]].size).to_a
 
     vars.each { |var|
       operation = nil
@@ -304,14 +307,25 @@ class ExtCsv < OpenStruct
         raise
       end
       #test stdout << "\n NEW VALUE :::::::::::::::\n"
-      obj_values = @table[var]
-      obj_values = [(0..obj_values.size-1).to_a, obj_values].transpose.values_at(*lookup)
-      
+      obj_values  = @table[var]
+      size        = @table[var].size
+      checkValues = [(0...size).to_a, obj_values].transpose
+      if ShunkSize < size
+        container = []
+        (0..size/ShunkSize).collect {|i|
+          checkValues.values_at(*(lookup[i*ShunkSize,ShunkSize]))
+        }.each {|v| v.each {|vv| container << vv} }
+        checkValues = container
+      else
+        checkValues = checkValues.values_at(*lookup)
+      end
+
       if operation.kind_of?(Regexp)
-        lookup = lookup & obj_values.find_all {|i,v| operation.match(v.to_s)}.transpose[0].to_a
+        lookup = lookup & checkValues.find_all {|i,v| operation.match(v.to_s)}.transpose[0].to_a
       else
-        lookup = lookup & obj_values.find_all {|i,v| 
-          next if v.nil? or v.empty?
+        lookup = lookup & checkValues.find_all {|i,v|
+          next if v.nil?
+          next if v.empty? if v.respond_to?(:empty?)
           v = "'" + v + "'" if type == :string
           #test $stdout <<[v,operation,value].join(" ") << "\n"
           eval([v,operation,value].join(" "))
@@ -381,7 +395,7 @@ class ExtCsv < OpenStruct
     return self.class.new("hash","plain",h)
   end
   def clear
-    @table.each {|k,v| @table[k] = [] if v.kind_of?(Array)} 
+    @table.each {|k,v| @table[k] = [] if v.kind_of?(Array)}
   end
   def empty?
     return true if @table.empty?

data/lib/extcsv_diagram.rb

@@ -109,7 +109,7 @@ module ExtCsvDiagram
   def ExtCsvDiagram.checkColumns(obj,*cols)
     cols.each {|col|
       next if col.kind_of?(Hash)
-      unless obj.datacolumns.include?(col)
+      unless obj.datacolumns.include?(col.to_s)
         print "[plot] Input data does NOT contain column '#{col.to_s}'\n"
         raise ArgumentError
       end
@@ -175,7 +175,7 @@ module ExtCsvDiagram
   def ExtCsvDiagram.plot_xy(obj,xColumn,yColumn,title,options={})
     checkColumns(obj,xColumn,yColumn) unless options[:skipColumnCheck]
     options        = GRAPH_OPTIONS.merge(options)
-    outputfilename = (options[:filename].nil?) ? obj.filename : options[:filename]
+    #outputfilename = (options[:filename].nil?) ? obj.filename : options[:filename]
     groupBy        = (options[:groupBy]).nil? ? [] : options[:groupBy]
     Gnuplot.open {|gp|
       Gnuplot::Plot.new(gp) {|plot|

data/test/test_extcsv.rb

@@ -50,7 +50,7 @@ class TestExtCsv < Test::Unit::TestCase
     assert_equal(org.x2,td_by_str.x2)
   end
   def test_datasets
-    test_simple = ExtCsv.new(IMPORT_TYPE,"txt",TEST_DATA)
+    test_simple = ExtCsv.new(IMPORT_TYPE,"tsv",TEST_DATA)
     assert_equal(["100.0", "950.0"], test_simple.datasets("col1","col2")[29])
   end
   def test_csv
@@ -228,9 +228,10 @@ class TestExtCsv < Test::Unit::TestCase
   end
   def test_clear
     simple = ExtCsv.new(IMPORT_TYPE,"txt",TEST_DATA)
-    assert_equal(false, simple.col1.empty?)
+    mycol = simple.datacolumns[0]
+    assert_equal(false, simple.send(mycol).empty?)
     simple.clear
-    assert(simple.col1.empty?)
+    assert_equal(true, simple.send(mycol).empty?)
     assert(simple.empty?)
   end
   def test_each_by
@@ -495,7 +496,7 @@ class TestExtCsv < Test::Unit::TestCase
     assert_nil(csv + simple0)
   end
   def test_version
-    assert_equal('0.12.0',ExtCsv::VERSION)
+    assert_equal('0.12.2',ExtCsv::VERSION)
   end
 
   def test_add
@@ -520,4 +521,16 @@ class TestExtCsv < Test::Unit::TestCase
     d = c.columns(*c.datacolumns[-3..-1])
     assert_equal(c.datasets(*d.datacolumns),d.datasets)
   end
+  def test_MRI_bug
+    # MRI has problems with long list of optional arguments like
+    # values_at(*ins) where ins.size > 160000 (happens with large objects
+    # jruby works though
+    size  = 2*10**5+1
+    limit = size/2
+    a = (0...size).to_a.map(&:to_s)
+    a = (0...size).to_a
+    obj = ExtCsv.new("hash","plain",{:a => a})
+    upper = obj.selectBy(:a => "> #{limit}")
+    assert_equal(limit,upper.size)
+  end
 end

data/test/test_extcsv_diagram.rb

@@ -29,14 +29,15 @@ class TestExtCsvDisplay < Test::Unit::TestCase
 
   def test_simple
     f=ExtCsv.new("file","txt",TEST_DATA)
-    ExtCsvDiagram.plot(f,["col4"],"col0",["col1"])
+    ExtCsvDiagram.plot_xy(f,"col5","col3",'') #,"col0",["col1"])
   end
-  def test_icon
+  def _test_icon
     icon = ExtCsv.new(IMPORT_TYPE,"psv",ICON)
 
     icon.datetime = []
     icon.date.each_with_index{|date,i| icon.datetime << [date,icon.time[i]].join(' ') }
 
+    puts "SIZE: #{icon.size}"
     [:date,:time,:depth,VAR.downcase.to_sym].each {|col| puts [col.to_s,icon.send(col).max].join('[max]: ') }
     ExtCsvDiagram.plot_xy(icon,"datetime",VAR.downcase,'ICON OCE_BASE: Mean. Temperatur (uneven levels, r8656, full run: 2001-2012)',
                        :label_position => 'below',:skipColumnCheck => true,
@@ -45,7 +46,7 @@ class TestExtCsvDisplay < Test::Unit::TestCase
 #                         :xrange => "",
                          :onlyGroupTitle => true,
 #                         :addSettings => ["logscale y"],
-                         :terminal => "png",
+#                         :terminal => "png",
                          :ylabel => "#{VAR} [degC]",
                          :input_time_format => "'%Y%m%d %H:%M:%S'",
                          :filename => "icon-OCE_BASE_uneven-r8656-fullrun-Mean#{VAR}",
@@ -110,13 +111,14 @@ class TestExtCsvDisplay < Test::Unit::TestCase
   end
   def test_extcsv_diagram_limits
     td = ExtCsv.new("file","txt",TEST_DATA_NEW)
+    td.add(:step,(1...td.size).to_a)
     ExtCsvDiagram.plot(td[0,21],
-                       [:col1],
-                       :index,
-                       [:col3],
-                       :zeit,
-                       [:col8],
-                       'limit test', 
+                       ["col1"],
+                       :step,
+                       ["col3"],
+                       "step",
+                       ["col8"],
+                       'limit test',
                        :y1limits       => [1.9],
                        :y1limitname    => "y1 Limit",
                        :y2limits       => [8.2],

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: extcsv
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.12.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-16 00:00:00.000000000 Z
+date: 2012-04-30 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: stark.dreamdetective@gmail.com