xcsv 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9a8d5c371cffac1168bab3828f1e7f3853a859d5c4eef6ef8cda2e45aee2ed5
-  data.tar.gz: a03cd965c98b332e6c4b3dff1181d1b1bb959852958239f5da2389d12fe473db
+  metadata.gz: 4ff9165c5564192166cd4dba6c141e7cbec377d7e89b1e83aefeeed16a5f14e1
+  data.tar.gz: a8d607ec64710a73488358e07481f99a62d12f2fb09fe9747b44893ebc6c6860
 SHA512:
-  metadata.gz: b8069138d85fea38fa096badd6055a37bfc3cc30dcf63d39240918fb65b282506352e1c95b8ae56e2dcb537399a0f9855cf84850680f978c1402e0baa61123c6
-  data.tar.gz: 5eb46dc0ddef820e462aa2bc0a2f4c8018517426238ecf8d33def906e1ff05e760edf2760be7fd73ee9b752a3c082291bfedffadc59215c29f591f10f7d66ff7
+  metadata.gz: 78a1b77ae1f099e37966c93fca1669feb1c02bdb0f8e760bd1b8db36b9484eb3180041f2b704bf7dd5ba948bf6f25683b7924c27bb358911a8bf534463711a93
+  data.tar.gz: f3bb8a94f289faf706a65576f7d51b36f2c13504c76dd874813e34ee1572217ab3f5c9be2ba8db9e7381108dfdad95734d93960632dd08c6f4a82f1d39dce5ca

data/Cargo.lock

@@ -1,8 +1,39 @@
+[[package]]
+name = "adler32"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "aho-corasick"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "memchr 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "build_const"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "cc"
+version = "1.0.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "cfg-if"
 version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "crc"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "cstr-macro"
 version = "0.1.0"
@@ -25,6 +56,16 @@ dependencies = [
  "memchr 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "flate2"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz-sys 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz_oxide_c_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "helix"
 version = "0.7.5"
@@ -35,6 +76,11 @@ dependencies = [
  "libcruby-sys 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "libc"
 version = "0.2.43"
@@ -58,11 +104,77 @@ dependencies = [
  "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "miniz-sys"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "miniz_oxide_c_api"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz_oxide 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "regex"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "aho-corasick 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex-syntax 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "serde"
 version = "1.0.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "thread_local"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "ucd-util"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "utf8-ranges"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "version_check"
 version = "0.1.5"
@@ -73,17 +185,34 @@ name = "xcsv"
 version = "0.1.0"
 dependencies = [
  "csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "flate2 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "helix 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [metadata]
+"checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c"
+"checksum aho-corasick 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1e9a933f4e58658d7b12defcf96dc5c720f20832deebe3e0a19efd3b6aaeeb9e"
+"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
+"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16"
 "checksum cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "082bb9b28e00d3c9d39cc03e64ce4cea0f1bb9b3fde493f0cbc008472d22bdf4"
+"checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb"
 "checksum cstr-macro 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db53fddba18cdd35477a7213a3ef6acfbfa333c31b42ce019e544c4a1420a06f"
 "checksum csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d54f6b0fd69128a2894b1a3e57af5849a0963c1cc77b165d30b896e40296452"
 "checksum csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4dd8e6d86f7ba48b4276ef1317edc8cc36167546d8972feb4a2b5fec0b374105"
+"checksum flate2 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3b0c7353385f92079524de3b7116cf99d73947c08a7472774e9b3b04bff3b901"
 "checksum helix 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)" = "49a017e3e798ad9386e0a0584e66fd6c04a80ccc1242eb8f689c62ce6f408240"
+"checksum lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a374c89b9db55895453a74c1e38861d9deec0b01b405a82516e9d5de4820dea1"
 "checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d"
 "checksum libcruby-sys 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fef6028cdce0c8d55676fd1d66bb810facef8cade0dd71d28511d375e84da4c0"
 "checksum memchr 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0a3eb002f0535929f1199681417029ebea04aadc0c7a4224b46be99c7f5d6a16"
+"checksum miniz-sys 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0300eafb20369952951699b68243ab4334f4b10a88f411c221d444b36c40e649"
+"checksum miniz_oxide 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5ad30a47319c16cde58d0314f5d98202a80c9083b5f61178457403dfb14e509c"
+"checksum miniz_oxide_c_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "28edaef377517fd9fe3e085c37d892ce7acd1fbeab9239c5a36eec352d8a8b7e"
+"checksum regex 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ee84f70c8c08744ea9641a731c7fadb475bf2ecc52d7f627feb833e0b3990467"
+"checksum regex-syntax 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fbc557aac2b708fe84121caf261346cc2eed71978024337e42eb46b8a252ac6e"
 "checksum serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "15c141fc7027dd265a47c090bf864cf62b42c4d228bbcf4e51a0c9e2b0d3f7ef"
+"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
+"checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86"
+"checksum utf8-ranges 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "796f7e48bef87609f7ade7e06495a87d5cd06c7866e6a5cbfceffc558a243737"
 "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"

data/Cargo.toml

@@ -9,3 +9,5 @@ crate-type = ["cdylib"]
 [dependencies]
 helix = "0.7.5"
 csv = "1.0.2"
+regex = "1.0.6"
+flate2 = "1.0.4"

data/README.md

@@ -1,21 +1,81 @@
 Fast CSV reader based on [Rust CSV crate](https://docs.rs/csv/1.0.2/csv/)
 
----
+### Installation
 
 1. Install [Rust](https://www.rust-lang.org/)
 
-`curl https://sh.rustup.rs -sSf | sh`
+   `curl https://sh.rustup.rs -sSf | sh`
 
-Don't miss this message:
+   Don't miss this message:
 
+	```
+	Rust is installed now. Great!
+
+	To get started you need Cargo's bin directory ($HOME/.cargo/bin) in your PATH 
+	environment variable. Next time you log in this will be done automatically.
+
+	To configure your current shell run source $HOME/.cargo/env
+
+	```
+
+2. `gem install xcsv`
+
+### Usage
+
+```ruby
+require 'xcsv'
+
+# Enumerable
+csv_reader = XSV.new("foo.csv")
+csv_reader.each do |rec|
+  rec #=> [col1, col2, col3, ...]
+end
+
+csv_reader = XSV.new("foo.csv")
+csv_reader.take(10).to_a #=> [[col1, ...], [col1, ...], ...]
+
+# While loop
+csv_reader = XSV.new("bar.csv") 
+while (rec = csv_reader.next) do
+  rec #=> [col1, col2, col3, ...]
+end
+
+# Both forms will gunzip if file name ends with .gz
+csv_reader = XSV.new("foo_bar.csv.gz") 
+while (rec = csv_reader.next) do
+  rec #=> [col1, col2, col3, ...]
+end
 ```
-Rust is installed now. Great!
 
-To get started you need Cargo's bin directory ($HOME/.cargo/bin) in your PATH 
-environment variable. Next time you log in this will be done automatically.
+### Benchmarks
+
+#### Code
+
+```ruby
+# FastestCSV
+FastestCSV.foreach('sample.csv') do |rec|
+end
 
-To configure your current shell run source $HOME/.cargo/env
+# XCSV
+csv_reader = XCSV.new('sample.csv')
+while (l = csv_reader.next) do
+end
 
+# CSV
+CSV.foreach('sample.csv') do |rec|
+end
 ```
 
-2. `bundle exec rake`
+#### Parameters
+
+|Records|File size|CPU|
+|---|---|---|
+|1M|742M (426M .gz)|i7-6600U @ 2.60GHz|
+
+#### Elapsed time (secs):
+
+|FastestCSV|XCSV|XCSV (.gz)|CSV|
+|---|---|---|---|
+|10.1|12.4|20.7|50.5|
+
+**Note**: FastestCSV doesn't decode embedded newlines

data/lib/xcsv/version.rb

@@ -1,3 +1,3 @@
 class XCSV
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/xcsv.rb

@@ -1,2 +1,12 @@
 require 'helix_runtime'
 require 'xcsv/native'
+
+class XCSV
+  include Enumerable
+
+  def each
+    while (r = self.next) do
+      yield r
+    end
+  end
+end

data/src/lib.rs

@@ -1,19 +1,25 @@
 extern crate csv;
+extern crate flate2;
+extern crate regex;
 
 #[macro_use]
 extern crate helix;
 
 use std::fs::File;
-use std::io::BufReader;
+use std::io::{BufReader, Read};
 use std::ops::{Deref, DerefMut};
 
+use flate2::read::GzDecoder;
+use regex::Regex;
+
 use helix::{FromRuby, CheckResult};
 use helix::sys::{VALUE};
 
 type CSVIterType = Iterator<Item=Result<csv::StringRecord, csv::Error>>;
 
 struct CSVIter {
-    iter: Box<CSVIterType>
+    iter: Box<CSVIterType>,
+    path: String,
 }
 
 impl Deref for CSVIter {
@@ -32,7 +38,7 @@ impl DerefMut for CSVIter {
 
 impl std::fmt::Debug for CSVIter {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, "CSVIter")
+        write!(f, "CSVIter: {}", self.path)
     }
 }
 
@@ -48,22 +54,29 @@ impl FromRuby for CSVIter {
     fn from_ruby(value: VALUE) -> CheckResult<CSVIter> {
         let checked_path = String::from_ruby(value)?;
 
-        let csv_reader = 
-            match File::open(String::from_checked(checked_path)) {
-                Ok(f) => 
-                    BufReader::new(f),
-                Err(e) =>
-                    raise!(format!("Error while opening file: {}", e)),
+        let path = String::from_checked(checked_path);
+
+        let gz_regex = Regex::new("\\.gz\\z").unwrap();
+
+        let buf_reader = 
+            match File::open(path.clone()) {
+                Ok(f)   => BufReader::new(f),
+                Err(e)  => raise!(format!("Error while opening file: {}", e)),
+            };
+
+        let gz_reader: Box<Read> =
+            if gz_regex.is_match(&path) {
+                Box::new(GzDecoder::new(buf_reader)) 
+            } else {
+                Box::new(buf_reader)  
             };
 
         let csv_reader =
             csv::ReaderBuilder::new()
                 .has_headers(false)
-                .from_reader(csv_reader);
-
-        let records = csv_reader.into_records();
+                .from_reader(gz_reader);
 
-        Ok(CSVIter{iter: Box::new(records)})
+        Ok(CSVIter{iter: Box::new(csv_reader.into_records()), path: path})
     }
 
     fn from_checked(checked: CSVIter) -> CSVIter {
@@ -81,7 +94,7 @@ ruby! {
             XCSV { helix, iter }
         }
 
-        def next_line(&mut self) -> Result<Option<Vec<String>>, helix::Error> {
+        def next(&mut self) -> Result<Option<Vec<String>>, helix::Error> {
             match self.iter.next() {
                 Some(Ok(record)) =>
                     Ok(Some(record.iter().map(|s| s.to_string()).collect())), 

data/xcsv.gemspec

@@ -22,6 +22,8 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = '>= 2.0'
+
   spec.add_runtime_dependency "helix_runtime", "= 0.7.5"
   spec.add_development_dependency "bundler", "~> 1.17"
   spec.add_development_dependency "rake", "~> 12.3"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xcsv
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Victor Moroz
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-16 00:00:00.000000000 Z
+date: 2018-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: helix_runtime
@@ -98,7 +98,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="