ccsv 0.1.2 → 1.0.1

data/CHANGELOG

@@ -1,3 +1,7 @@
+v1.0.1 Tests moved to minitest (zhum).
+
+v1.0.0 Allow custom delimiter and integer range filtration (zhum).
+
 v0.1.2. Allow lines of any length (crossman).
 
 v0.1.1 Remove env.h and node.h for compilation for ruby 1.9.3 (pronix).

data/Gemfile

@@ -0,0 +1,7 @@
+group :test do
+	gem 'minitest'
+end
+
+group :development do
+	gem 'echoe'
+end

data/Gemfile.lock

@@ -0,0 +1,23 @@
+GEM
+  specs:
+    allison (2.0.3)
+    echoe (4.6.5)
+      allison (>= 2.0.3)
+      rake (>= 0.9.2)
+      rdoc (>= 2.5.11)
+      rubyforge (>= 2.0.4)
+    json (1.8.1)
+    json_pure (1.8.1)
+    minitest (5.2.1)
+    rake (10.1.0)
+    rdoc (4.0.1)
+      json (~> 1.4)
+    rubyforge (2.0.4)
+      json_pure (>= 1.1.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  echoe
+  minitest

data/Manifest

@@ -1,11 +1,12 @@
 CHANGELOG
+Gemfile
+Gemfile.lock
 LICENSE
 Manifest
 README.rdoc
 Rakefile
+compile
 ext/ccsv.c
 ext/ccsv.h
 ext/extconf.rb
-test/data.csv
-test/data_small.csv
-test/unit/test_ccsv.rb
+spec/ccsv_spec.rb

data/README.rdoc

@@ -3,13 +3,40 @@ Ccsv
 
 A pure-C CSV parser.
 
-for after make changes - add description to CHANGELOG and run rake build
+== Installation
 
-== License
+  gem install ccsv
+
+== Usage
+
+  require 'rubygems'
+  require 'ccsv'
+
+  Ccsv.foreach("data.csv") do |line|
+    # Do something with the line array
+  end
+
+  # print all logins
+  Ccsv.foreach("/etc/passwd",":") do |line|
+    puts line[0]
+  end
 
-Copyright 2007, 2008 Cloudburst, LLC. Licensed under the AFL 3. See the included LICENSE file.
+== Advanced usage
 
-The public certificate for the gem is here[http://blog.evanweaver.com/files/evan_weaver-original-public_cert.pem].
+Get users from passwd file, with UIDs between 1000 and 1010 and between 2000 and 2010.
+Works with numbers only!
+Third argument is column index, used for filtering, then one or more intervals.
+
+  Ccsv.foreach("/etc/passwd",":",2,[1000..1010],[2000..2010]) do |line|
+    puts line[0]
+  end
+
+== Contacts
+
+  Forks, pull-requests and other contacts via guthub: http://github.com/evan/ccsv/
+
+== License
 
-If you use this software, please {make a donation}[http://blog.evanweaver.com/donate/], or {recommend Evan}[http://www.workingwithrails.com/person/7739-evan-weaver] at Working with Rails.
+Copyright 2012-2013 Sergey Zhumatiy
 
+Copyright 2007-2012 Cloudburst, LLC. Licensed under the AFL 3. See the included LICENSE file.

data/Rakefile

@@ -1,9 +1,19 @@
+require 'bundler'
+Bundler.require(:default, :development)
+
 require 'echoe'
 
 Echoe.new("ccsv") do |p|
-  p.author = "Evan Weaver"
+  p.author = ["Evan Weaver","Sergey Zhumatiy"]
   p.project = "evan"
+  p.email = "serg@parallel.ru"
   p.summary = "A pure-C CSV parser."
+  p.description = "Ruby CSV parser gem, written in pure C."
   p.url = "http://github.com/evan/ccsv/"
   p.docs_host = "evan.github.com/fauna/"
 end
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+	  t.pattern = "spec/*_spec.rb"
+end

data/ccsv.gemspec

@@ -1,26 +1,23 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = %q{ccsv}
-  s.version = "0.1.2"
+  s.name = "ccsv"
+  s.version = "1.0.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
-  s.authors = [%q{Evan Weaver}]
-  s.cert_chain = [%q{/Users/eweaver/p/configuration/gem_certificates/evan_weaver-original-public_cert.pem}]
-  s.date = %q{2012-02-24}
-  s.description = %q{A pure-C CSV parser.}
-  s.email = %q{}
-  s.extensions = [%q{ext/extconf.rb}]
-  s.extra_rdoc_files = [%q{CHANGELOG}, %q{LICENSE}, %q{README.rdoc}, %q{ext/ccsv.c}, %q{ext/ccsv.h}, %q{ext/extconf.rb}]
-  s.files = [%q{CHANGELOG}, %q{LICENSE}, %q{Manifest}, %q{README.rdoc}, %q{Rakefile}, %q{ext/ccsv.c}, %q{ext/ccsv.h}, %q{ext/extconf.rb}, %q{test/data.csv}, %q{test/data_small.csv}, %q{test/unit/test_ccsv.rb}, %q{ccsv.gemspec}]
-  s.homepage = %q{http://github.com/evan/ccsv/}
-  s.rdoc_options = [%q{--line-numbers}, %q{--inline-source}, %q{--title}, %q{Ccsv}, %q{--main}, %q{README.rdoc}]
-  s.require_paths = [%q{lib}, %q{ext}]
-  s.rubyforge_project = %q{evan}
-  s.rubygems_version = %q{1.8.6}
-  s.signing_key = %q{/Users/eweaver/p/configuration/gem_certificates/evan_weaver-original-private_key.pem}
-  s.summary = %q{A pure-C CSV parser.}
-  s.test_files = [%q{test/unit/test_ccsv.rb}]
+  s.authors = ["Evan Weaver, Sergey Zhumatiy"]
+  s.date = "2014-01-10"
+  s.description = "Ruby CSV parser gem, written in pure C."
+  s.email = "serg@parallel.ru"
+  s.extensions = ["ext/extconf.rb"]
+  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README.rdoc", "ext/ccsv.c", "ext/ccsv.h", "ext/extconf.rb"]
+  s.files = ["CHANGELOG", "Gemfile", "Gemfile.lock", "LICENSE", "Manifest", "README.rdoc", "Rakefile", "compile", "ext/ccsv.c", "ext/ccsv.h", "ext/extconf.rb", "spec/ccsv_spec.rb", "ccsv.gemspec"]
+  s.homepage = "http://github.com/evan/ccsv/"
+  s.rdoc_options = ["--line-numbers", "--title", "Ccsv", "--main", "README.rdoc"]
+  s.require_paths = ["lib", "ext"]
+  s.rubyforge_project = "evan"
+  s.rubygems_version = "1.8.23"
+  s.summary = "A pure-C CSV parser."
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/compile

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+bundle install
+rake compile
+rake build
+rake gem
+rake test

data/ext/ccsv.c

@@ -1,32 +1,163 @@
+#include <limits.h>
 #include "ccsv.h"
 
 static VALUE rb_cC;
 
-static VALUE foreach(VALUE self, VALUE filename) {
+/* Ccsv.foreach(filename,delimiter,[index],[range,...]) do |line| ... */
 
-  FILE *file = fopen(StringValueCStr(filename), "r");
-  if (file == NULL)
-    rb_raise(rb_eRuntimeError, "File not found");
+struct pair_st {
+  long int low,high;
+};
 
+#define MAX_INTERVALS 1024
+
+static VALUE foreach(int argc, VALUE* argv, VALUE self) {
+  char DELIM=DEF_DELIM;
   char *line = NULL;
   size_t len = 0;
-  char *token;
-  int idx;
+  char *token,*start,*nobackslash,*t2, *str;
+  int idx,count,pairs_count,searchfield,flag,i,array_length,range_i,len2;
+  long check;
+  FILE *file;
+  ID min_method, max_method;
+  VALUE min_val, max_val;
+  VALUE tmp_value, rest_args, filename;
+  ID    array_length_method; /*----------------------------------------*/
+  struct pair_st pairs[MAX_INTERVALS];
 
   VALUE ary;
-  
+
+  rb_scan_args(argc,argv,"1*", &filename, &rest_args);
+
+/*  if (argc == 0) {  // there should only be 1 or 2 arguments
+    rb_raise(rb_eArgError, "wrong number of arguments");
+  }
+
+  file = fopen(StringValueCStr(argv[0]), "r");
+  if (file == NULL)
+    rb_raise(rb_eRuntimeError, "File not found");
+*/
+  file = fopen(StringValueCStr(filename), "r");
+  if(file==NULL){
+    rb_raise(rb_eRuntimeError, "File not found");
+  }
+
+  if (argc >1 ) { /* delimiter */
+    tmp_value=rb_ary_entry(rest_args,0);
+    str=StringValueCStr(tmp_value);
+    DELIM=str[0];
+  }
+
+  if (argc >2 ) { /* search index */
+    tmp_value=rb_ary_entry(rest_args,1);
+    searchfield=NUM2INT(tmp_value);
+  }
+  else{
+    searchfield=-1;
+  }
+
+  min_val=rb_funcall(rest_args,rb_intern("length"), 0);
+  array_length=NUM2INT(min_val);
+  /*rb_warn("Length=%d",array_length);*/
+
+  min_method = rb_intern("min");
+  max_method = rb_intern("max");
+  /*------------test_id    = rb_intern("class");*/
+  range_i=0;
+  for(idx=2;idx<array_length;++idx){
+    min_val=rb_funcall(rb_ary_entry(rest_args,idx),rb_intern("length"), 0);
+    len2=NUM2INT(min_val);
+    for(i=0;i<len2;++i){
+      VALUE e=rb_ary_entry(rb_ary_entry(rest_args,idx),i);
+      if(range_i>MAX_INTERVALS)
+        rb_raise(rb_eRuntimeError, "Too much ranges passed");
+      if(TYPE(e) == T_NIL){
+        pairs[range_i].low=LONG_MIN;
+        pairs[range_i].high=LONG_MAX;
+
+        continue; /* just skip nil */
+      }
+      if (! (rb_respond_to(e, min_method) & rb_respond_to(e, max_method)))
+        rb_raise(rb_eRuntimeError, "Not range passed to Ccsv.foreach");
+
+      min_val=rb_funcall(e, min_method, 0);
+      max_val=rb_funcall(e, max_method, 0);
+  /*    rb_warn("!\n");*/
+      pairs[range_i].low=NUM2LONG(min_val);
+      /*rb_warn("2\n");*/
+      pairs[range_i].high=NUM2LONG(max_val);
+      /*rb_warn("RANGE: %ld .. %ld (%d)\n",(long)pairs[range_i].low,(long)pairs[range_i].high,(int)(range_i));*/
+      range_i++;
+    }
+  }
+  pairs_count=range_i;
+
   while (getline(&line, &len, file) != -1) {
+    /* chomp! */
+    if(token=index(line,EOL)){
+      *token='\0';
+    }
+    /*rb_warning("4\n");*/
     ary = rb_ary_new();
-    token = strtok(line, DELIMITERS);
+    start=line;
+    nobackslash=line;
+    while(token=index(nobackslash, DELIM)){
+/*      rb_warning("5\n");*/
+      count=0;
+      t2=token-1;
+      while((t2>=line) && (*t2=='\\'))
+        {++count;--t2;}
+      if(count%2 ==1){ /* backslashed! skip */
+        nobackslash=token;
+        continue;
+      }
+      break;
+    }
     idx = 0;
+    flag=1;
 
     while (token != NULL) {
-      rb_ary_store(ary, idx, rb_str_new(token, strlen(token)));
-      idx ++;
-      token = strtok(NULL, DELIMITERS);
+      *token='\0';
+      if(searchfield==idx){
+        flag=0;
+        /* do check! */
+        sscanf(start,"%ld",&check);
+        for(i=0;i<pairs_count;++i){
+          /*rb_warn("check %ld: [%ld .. %ld]",check,pairs[i].low,pairs[i].high);*/
+          if(pairs[i].low<check && pairs[i].high>check){
+            /*rb_warn("check passed");*/
+            flag=1; /* yahooo! */
+            break;
+          }
+        }
+      }
+
+      /* not in ranges! */
+      if(flag==0)
+        break;
+
+      rb_ary_store(ary, idx, rb_str_new(start, token-start));
+      idx++;
+      nobackslash=start=token+1;
+      while(token=index(nobackslash, DELIM)){
+        count=0;
+        t2=token-1;
+        while((t2>=line) && (*t2=='\\'))
+          {++count;--t2;}
+        if(count%2 ==1){ /* backslashed! skip */
+          nobackslash=token;
+          continue;
+        }
+        break;
+      }
     }
+    if(flag==0)
+      continue;
 
+    /* last item */
+    rb_ary_store(ary, idx, rb_str_new(start, strlen(start)));
     /* OBJ_FREEZE(ary); */
+
     rb_yield(ary);
     /* FL_UNSET((ary), FL_FREEZE); */
 
@@ -45,5 +176,7 @@ void
 Init_ccsv()
 {
   rb_cC = rb_define_class("Ccsv", rb_cObject);
-  rb_define_singleton_method(rb_cC, "foreach", foreach, 1);
+  rb_define_singleton_method(rb_cC, "foreach", foreach, -1);
+  rb_define_const(rb_cC, "MAX", LONG2NUM(LONG_MAX));
+  rb_define_const(rb_cC, "MIN", LONG2NUM(LONG_MIN));
 }

data/ext/ccsv.h

@@ -1,4 +1,7 @@
 #include "ruby.h"
 
-#define DELIMITERS ",\n"
+/*#define DELIMITERS ",\n"*/
+
+#define EOL '\n'
+#define DEF_DELIM ','
 #define ARY_DEFAULT_SIZE 16

data/spec/ccsv_spec.rb

@@ -0,0 +1,134 @@
+require 'bundler'
+Bundler.require(:default, :test)
+
+gem 'minitest'
+#require 'minitest/benchmark'
+require 'minitest/autorun'
+#require 'minitest/spec'
+require 'ccsv'
+#require 'csv'
+
+TEST_CSV="/tmp/test.csv"
+
+module CSVScan
+  def self.foreach(file, &block)
+    open(file) do |f|
+      scan(f, &block)
+    end
+  end
+end
+
+def create_csv(delimiter=',')
+	open(TEST_CSV,"w") do |f|
+        	1.upto(100000) do |n|
+			f.puts [n,2*n,3+n].join(delimiter)
+		end
+	end
+end
+
+
+describe Ccsv do
+	before do
+		@csv=[]
+	end
+
+	it 'reads csv with default delimiter' do
+		create_csv
+		Ccsv.foreach(TEST_CSV) do |v|
+			@csv << v
+		end
+		@csv[15000].must_equal(['15001','30002','15004'])
+		@csv.size.must_equal(100000)
+	end
+
+	it 'reads csv with tab delimiter' do
+		create_csv("\t")
+		Ccsv.foreach(TEST_CSV,"\t") do |v|
+			@csv << v
+		end
+		@csv[15000].must_equal(['15001','30002','15004'])
+	end
+
+	it 'reads csv with comma delimiter' do
+		create_csv(',')
+		Ccsv.foreach(TEST_CSV,',') do |v|
+			@csv << v
+		end
+		@csv[15000].must_equal(['15001','30002','15004'])
+	end
+
+	it 'raises error' do
+		proc {
+			Ccsv.foreach('/non-existent-file') do |x| end
+		}.must_raise(RuntimeError)
+	end
+
+#	bench_performance_linear 'just read', 0.9 do |n|
+#		create_csv(',',n)
+#		Ccsv.foreach(TEST_CSV)
+#	end
+end
+
+#describe 'my benchmarks' do
+#end
+
+__END__
+require 'test/unit'
+require 'ccsv'
+require 'benchmark'
+
+# Yeah, I know.
+begin
+  require 'csv'
+  require 'rubygems'
+  require 'lightcsv'
+  require 'csvscan'
+
+  module CSVScan
+    def self.foreach(file, &block)
+      open(file) do |f|
+        scan(f, &block)
+      end
+    end
+  end
+
+rescue LoadError
+end
+
+class TestCcsv < Test::Unit::TestCase
+
+  def setup
+    @dir = "#{File.dirname(__FILE__)}/../"
+  end
+
+  def test_should_raise
+    assert_raises(RuntimeError) do
+      Ccsv.foreach('fdssfd') do
+      end
+    end
+  end
+
+  def test_accuracy
+    ccsv = []
+    file = @dir + "data_small.csv"
+    Ccsv.foreach(file) do |values|
+      ccsv << values.dup
+    end
+    csv = []
+    CSV.foreach(file) do |values|
+      csv << values
+    end
+    assert_equal csv, ccsv
+  end
+
+  def test_speed
+    Benchmark.bm(5) do |x|
+      [Ccsv, CSV].each do |klass| # CSVScan, LightCsv,
+        x.report(klass.name) do
+          klass.foreach(@dir + "data.csv") do |values| end
+        end
+      end
+    end
+  end
+
+end