rscsv 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49902e25dff6b9904a52ad232077add5d62f6fa7
-  data.tar.gz: 11c0c12ca451c8e8d67da7bc4868e914bbf62436
+  metadata.gz: cb17bc325379d4ebf7d2c7e088b84495253c62d4
+  data.tar.gz: e379a8d05d9a39d551e8a118ace70c394598e187
 SHA512:
-  metadata.gz: 8f371fd7e2301db02e51516e648f5abc0add03d2a459a0da8600488f7a8a8ba39206fe8dae9ac644288c3504aa9d6d945f792c99cb4fc6bb9e9a3a236ea7dd56
-  data.tar.gz: 9051861c418781ea1e4aa862f070cb3c80201ba0fb2239c280957cabef721807c4b5a858621a2fa648e15b971ce2b6fc02727c2c6329c1b7642cee3fbf4b3b59
+  metadata.gz: 3fb8a756221d7b8e293c11ca885e2d935d90ea3abda2b451e0cbd29dd25d63cf2602e0d0109a01239935363fcc822c741b74750727eb29acbe458a95a7e85787
+  data.tar.gz: 13dfae6dee17a62a51cd64dcb2503f2472066db729d05fd925b01ddb304bb61a551a591cf2813c92b9739b358eb3bd28459e70f8f16411a8ff1f7f734248c367

data/README.md

@@ -24,8 +24,13 @@ Rscsv::Writer.generate_lines([['1', '2', '3'], ['3', '4', '5']])
 Rscsv::Writer.generate_line(['1', '2', '3'])
 # => 1,2,3\n
 
-Rscsv::Reader.parse("1,2,3\n4,5,6")
+Rscsv::Reader.parse("1,2,3\n4,5,6\n")
 # => [["1", "2", "3"], ["4", "5", "6"]]
+
+# Streaming from Enumerator
+Rscsv::Reader.each(["1,2,3\n","4,5,6\n"].each) do |row|
+  # yields ["1", "2", "3"] and ["4", "5", "6"]
+end
 ```
 
 This is ~3x faster than using native Ruby `CSV.generate` or `CSV.parse`.

data/lib/rscsv.rb

@@ -4,5 +4,13 @@ require 'rscsv/version'
 
 module Rscsv
   Reader = RscsvReader
+
+  class Reader
+    def self.each(input, &block)
+      each_internal(input, &block)
+    rescue StopIteration
+      nil
+    end
+  end
   Writer = RscsvWriter
 end

data/lib/rscsv/version.rb

@@ -1,3 +1,3 @@
 module Rscsv
-  VERSION = '0.2.1'.freeze
+  VERSION = '0.3.0'.freeze
 end

data/src/lib.rs

@@ -3,9 +3,12 @@ extern crate helix;
 extern crate csv;
 
 use std::error::Error;
+use std::io::Read;
+use std::slice::from_raw_parts;
 use helix::sys;
-use helix::sys::VALUE;
+use helix::sys::{VALUE, ID};
 use helix::{UncheckedValue, CheckResult, CheckedValue, ToRust, ToRuby};
+use helix::libc::c_int;
 
 struct VecWrap<T>(Vec<T>);
 
@@ -14,10 +17,8 @@ impl<T> UncheckedValue<VecWrap<T>> for VALUE
 {
     fn to_checked(self) -> CheckResult<VecWrap<T>> {
         if unsafe { sys::RB_TYPE_P(self, sys::T_ARRAY) } {
-            let len = unsafe { sys::RARRAY_LEN(self) };
-            let ptr = unsafe { sys::RARRAY_PTR(self) };
-            for i in 0..len {
-                let val = unsafe { *ptr.offset(i) };
+            let slice = ruby_array_to_slice(self);
+            for val in slice.iter() {
                 if let Err(error) = val.to_checked() {
                     return Err(format!("Failed to convert value for Vec<T>: {}", error));
                 }
@@ -31,20 +32,23 @@ impl<T> UncheckedValue<VecWrap<T>> for VALUE
     }
 }
 
+fn ruby_array_to_slice<'a>(array: VALUE) -> &'a [VALUE] {
+    let length = unsafe { sys::RARRAY_LEN(array) } as usize;
+    unsafe { from_raw_parts(sys::RARRAY_CONST_PTR(array), length) }
+}
+
 impl ToRust<VecWrap<String>> for CheckedValue<VecWrap<String>>
     where VALUE: UncheckedValue<String>,
           CheckedValue<String>: ToRust<String>
 {
     fn to_rust(self) -> VecWrap<String> {
-        let len = unsafe { sys::RARRAY_LEN(self.inner) };
-        let ptr = unsafe { sys::RARRAY_PTR(self.inner) };
-        let mut vec: Vec<String> = Vec::with_capacity(len as usize);
-        for i in 0..len {
-            let val = unsafe { *ptr.offset(i) };
+        let slice = ruby_array_to_slice(self.inner);
+        let mut vec: Vec<String> = Vec::with_capacity(slice.len());
+        for val in slice.iter() {
             let checked = val.to_checked().unwrap();
             vec.push(checked.to_rust());
         }
-        return VecWrap(vec);
+        VecWrap(vec)
     }
 }
 
@@ -53,15 +57,13 @@ impl ToRust<VecWrap<VecWrap<String>>> for CheckedValue<VecWrap<VecWrap<String>>>
           CheckedValue<VecWrap<String>>: ToRust<VecWrap<String>>
 {
     fn to_rust(self) -> VecWrap<VecWrap<String>> {
-        let len = unsafe { sys::RARRAY_LEN(self.inner) };
-        let ptr = unsafe { sys::RARRAY_PTR(self.inner) };
-        let mut vec: Vec<VecWrap<String>> = Vec::with_capacity(len as usize);
-        for i in 0..len {
-            let val = unsafe { *ptr.offset(i) };
+        let slice = ruby_array_to_slice(self.inner);
+        let mut vec: Vec<VecWrap<String>> = Vec::with_capacity(slice.len());
+        for val in slice.iter() {
             let checked = val.to_checked().unwrap();
             vec.push(checked.to_rust());
         }
-        return VecWrap(vec);
+        VecWrap(vec)
     }
 }
 
@@ -70,6 +72,9 @@ extern "C" {
     pub fn rb_ary_new_capa(capa: isize) -> VALUE;
     pub fn rb_ary_entry(ary: VALUE, offset: isize) -> VALUE;
     pub fn rb_ary_push(ary: VALUE, item: VALUE) -> VALUE;
+    pub fn rb_block_given_p() -> c_int;
+    pub fn rb_yield(value: VALUE);
+    pub fn rb_funcall(value: VALUE, name: ID, nargs: c_int, ...) -> VALUE;
 }
 
 impl ToRuby for VecWrap<csv::StringRecord> {
@@ -96,26 +101,137 @@ fn generate_lines(rows: VecWrap<VecWrap<String>>) -> Result<String, Box<Error>>
         wtr.write_record(&(row.0))?;
     }
 
-    return Ok(String::from_utf8(wtr.into_inner()?)?);
+    Ok(String::from_utf8(wtr.into_inner()?)?)
 }
 
-fn parse_csv(data: String) -> Result<Vec<csv::StringRecord>, csv::Error> {
-    let mut reader = csv::ReaderBuilder::new()
+fn record_to_ruby(record: &csv::ByteRecord) -> VALUE {
+    let inner_array = unsafe { rb_ary_new_capa(record.len() as isize) };
+    for column in record.iter() {
+        unsafe {
+            let column_value = sys::rb_utf8_str_new(column.as_ptr() as *const i8,
+                                                    column.len() as i64);
+            rb_ary_push(inner_array, column_value);
+        }
+    }
+    inner_array
+}
+
+
+impl UncheckedValue<Enumerator> for VALUE {
+    fn to_checked(self) -> CheckResult<Enumerator> {
+        Ok(unsafe { CheckedValue::new(self) })
+    }
+}
+
+impl ToRust<Enumerator> for CheckedValue<Enumerator> {
+    fn to_rust(self) -> Enumerator {
+        Enumerator { value: self.inner }
+    }
+}
+
+struct Enumerator {
+    value: VALUE,
+}
+
+struct EnumeratorRead {
+    value: VALUE,
+    next: Option<Vec<u8>>,
+}
+
+impl EnumeratorRead {
+    fn new(value: VALUE) -> EnumeratorRead {
+        EnumeratorRead {
+            value: value,
+            next: None,
+        }
+    }
+
+    fn read_and_store_overflow(&mut self, buf: &mut [u8], value: &[u8]) -> std::io::Result<usize> {
+        if value.len() > buf.len() {
+            match value.split_at(buf.len()) {
+                (current, next) => {
+                    for (index, c) in current.iter().enumerate() {
+                        buf[index] = *c;
+                    }
+                    self.next = Some(next.to_vec());
+                    Ok(current.len())
+                }
+            }
+
+        } else {
+            for (index, value) in value.iter().enumerate() {
+                buf[index] = *value;
+            }
+            self.next = None;
+            Ok(value.len() as usize)
+        }
+    }
+
+    fn read_from_external(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        let next = unsafe {
+            rb_funcall(self.value,
+                       sys::rb_intern("next\0".as_ptr() as *const i8),
+                       0)
+        };
+        let slice = unsafe {
+            from_raw_parts(sys::RSTRING_PTR(next) as *const u8,
+                           sys::RSTRING_LEN(next) as usize)
+        };
+
+        self.read_and_store_overflow(buf, slice)
+    }
+}
+
+impl Read for EnumeratorRead {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        match self.next.clone() {
+            Some(inner) => self.read_and_store_overflow(buf, &inner),
+            None => self.read_from_external(buf),
+        }
+    }
+}
+
+fn csv_reader<R: Read>(reader: R) -> csv::Reader<R> {
+    csv::ReaderBuilder::new()
+        .buffer_capacity(16 * 1024)
         .has_headers(false)
-        .from_reader(data.as_bytes());
-    let records = reader
+        .from_reader(reader)
+}
+
+fn yield_csv(data: Enumerator) -> Result<(), csv::Error> {
+    let mut reader = csv_reader(EnumeratorRead::new(data.value));
+    let mut record = csv::ByteRecord::new();
+
+    while reader.read_byte_record(&mut record)? {
+        let inner_array = record_to_ruby(&record);
+        unsafe {
+            rb_yield(inner_array);
+        }
+    }
+
+    Ok(())
+}
+
+fn parse_csv(data: String) -> Result<Vec<csv::StringRecord>, csv::Error> {
+    let mut reader = csv_reader(data.as_bytes());
+    reader
         .records()
-        .collect::<Result<Vec<csv::StringRecord>, csv::Error>>();
-    return records;
+        .collect::<Result<Vec<csv::StringRecord>, csv::Error>>()
 }
 
 ruby! {
     class RscsvReader {
+        def each_internal(data: Enumerator) {
+            match yield_csv(data) {
+                Err(_) => throw!("Error parsing CSV"),
+                Ok(_) => ()
+            }
+        }
         def parse(data: String) -> VecWrap<csv::StringRecord> {
             match parse_csv(data) {
                 Err(_) => throw!("Error parsing CSV"),
-                Ok(result) => return VecWrap(result)
-            };
+                Ok(result) => VecWrap(result)
+            }
         }
     }
     class RscsvWriter {
@@ -124,8 +240,8 @@ ruby! {
             let result = wtr.write_record(&(row.0));
             match result {
                 Err(_) => throw!("Error generating csv"),
-                Ok(_) => return String::from_utf8(wtr.into_inner().unwrap()).unwrap(),
-            };
+                Ok(_) => String::from_utf8(wtr.into_inner().unwrap()).unwrap(),
+            }
         }
 
         def generate_lines(rows: VecWrap<VecWrap<String>>) -> String {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rscsv
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Ville Lautanala
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-25 00:00:00.000000000 Z
+date: 2017-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: helix_runtime