osv 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c67f4e48abcf76bbbe0fce6513b67464ed550287451f5fa2b4c6e8f83256fef5
-  data.tar.gz: b38342f6e6c43050b45c99ea7f3fba4334423cb10d9df39b23ed054264b35e84
+  metadata.gz: a1b0c347b0bab5c9d31069c56f47999bfa51e85dfc1e127d1c4474a84ac19c53
+  data.tar.gz: 847b199da27b7c1329c1fa64fc8636592f004e93a12fc2ddd8db6127298ac23d
 SHA512:
-  metadata.gz: 190d1cfafc93554e5096682f1ce3ec6e9d8049b2669c471db5b27d9190a36500465d798a07e279d73d6e8569c47eaa259f48644a8f1df70138c6f5fc7a826348
-  data.tar.gz: f3c4ed3eac64aaf95e14bf7f5e60b84f2d698db586a2b8c5c6328389e0c067df298b070d4ac78eb91df444ebf23bad2e8ec74f5e7f3314f6ff0c8a248e52b263
+  metadata.gz: 62fa77c1ca98031f483569a4dba7cf9e4eca52a4b5fae293d274d5f89c48003e301eab01d95116cfc9cc6a2642e742d16046231a21d25e4a5143bd6ec3b40dac
+  data.tar.gz: 3832cbb6ebadfc718a8a5d1963de960ed3abf09d4559d2f0ffe031c642a6c2581dc6ad6edf5d65f22248812585ca464916375503753831024fc355fe4cd04455

data/Cargo.lock

@@ -11,6 +11,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "autocfg"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
+
 [[package]]
 name = "bindgen"
 version = "0.69.5"
@@ -90,6 +96,12 @@ version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
+[[package]]
+name = "futures-core"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
+
 [[package]]
 name = "glob"
 version = "0.3.1"
@@ -111,6 +123,16 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
 
+[[package]]
+name = "kanal"
+version = "0.1.0-pre8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b05d55519627edaf7fd0f29981f6dc03fb52df3f5b257130eb8d0bf2801ea1d7"
+dependencies = [
+ "futures-core",
+ "lock_api",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -139,6 +161,16 @@ dependencies = [
  "windows-targets",
 ]
 
+[[package]]
+name = "lock_api"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
 [[package]]
 name = "magnus"
 version = "0.6.4"
@@ -201,6 +233,7 @@ name = "osv"
 version = "0.1.0"
 dependencies = [
  "csv",
+ "kanal",
  "magnus 0.7.1",
  "rb-sys",
  "serde",
@@ -296,6 +329,12 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
 [[package]]
 name = "seq-macro"
 version = "0.3.5"

data/ext/osv/Cargo.toml

@@ -12,3 +12,4 @@ magnus = { version = "0.7", features = ["rb-sys"] }
 rb-sys = "0.9"
 serde = { version = "1.0", features = ["derive"] }
 serde_magnus = "0.8.1"
+kanal = "0.1.0-pre8"

data/ext/osv/src/lib.rs

@@ -1,9 +1,9 @@
-use magnus::{
-    block::Yield,
-    prelude::*,
-    scan_args::{get_kwargs, scan_args},
-    Error, RString, Ruby, Value,
-};
+mod reader;
+mod utils;
+
+use crate::reader::*;
+
+use magnus::{Error, Ruby};
 
 /// Initializes the Ruby extension and defines methods.
 #[magnus::init]
@@ -13,165 +13,3 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     module.define_module_function("for_each_compat", magnus::method!(parse_compat, -1))?;
     Ok(())
 }
-
-/// Helper function to get a readable from either an IO object or a file path
-fn get_readable(ruby: &Ruby, to_read: Value) -> Result<Box<dyn Read>, Error> {
-    if to_read.is_kind_of(ruby.class_io()) {
-        let reader = RubyIOReader::new(ruby, to_read)?;
-        Ok(Box::new(reader))
-    } else {
-        let path = to_read.to_r_string()?.to_string()?;
-        let file = std::fs::File::open(&path).map_err(|e| {
-            Error::new(
-                ruby.exception_runtime_error(),
-                format!("Failed to open file: {}", e),
-            )
-        })?;
-        Ok(Box::new(file))
-    }
-}
-
-/// Helper function to create a CSV reader with the given configuration
-fn create_csv_reader(
-    ruby: &Ruby,
-    to_read: Value,
-    has_headers: bool,
-    delimiter: Option<String>,
-) -> Result<csv::Reader<Box<dyn Read>>, Error> {
-    let readable = get_readable(ruby, to_read)?;
-    let delimiter = delimiter.unwrap_or_else(|| ",".to_string());
-
-    let rdr = csv::ReaderBuilder::new()
-        .has_headers(has_headers)
-        .delimiter(delimiter.as_bytes()[0])
-        .from_reader(readable);
-
-    Ok(rdr)
-}
-
-/// Common setup for CSV parsing, returns the reader and headers
-fn setup_csv_parser(
-    ruby: &Ruby,
-    to_read: Value,
-    has_headers: bool,
-    delimiter: Option<String>,
-) -> Result<(csv::Reader<Box<dyn Read>>, Vec<String>), Error> {
-    let mut rdr = create_csv_reader(ruby, to_read, has_headers, delimiter)?;
-
-    let first_row = rdr.headers().unwrap().clone();
-    let num_fields = first_row.len();
-
-    let headers = if has_headers {
-        first_row.iter().map(|h| h.to_string()).collect()
-    } else {
-        (0..num_fields).map(|i| format!("c{}", i)).collect()
-    };
-
-    Ok((rdr, headers))
-}
-
-/// Parse common arguments for CSV parsing
-fn parse_csv_args(args: &[Value]) -> Result<(Value, bool, Option<String>), Error> {
-    let parsed_args = scan_args::<(Value,), (), (), (), _, ()>(args)?;
-    let (to_read,) = parsed_args.required;
-
-    let kwargs = get_kwargs::<_, (), (Option<bool>, Option<String>), ()>(
-        parsed_args.keywords,
-        &[],
-        &["has_headers", "delimiter"],
-    )?;
-
-    let has_headers = kwargs.optional.0.unwrap_or(true);
-
-    Ok((to_read, has_headers, kwargs.optional.1))
-}
-
-/// Parses CSV data from a file and yields each row as a hash to the block.
-fn parse_csv(
-    ruby: &Ruby,
-    rb_self: Value,
-    args: &[Value],
-) -> Result<Yield<impl Iterator<Item = std::collections::HashMap<String, String>>>, Error> {
-    if !ruby.block_given() {
-        return Ok(Yield::Enumerator(rb_self.enumeratorize("for_each", args)));
-    }
-
-    let (to_read, has_headers, delimiter) = parse_csv_args(args)?;
-    let (rdr, headers) = setup_csv_parser(ruby, to_read, has_headers, delimiter)?;
-
-    let iter = rdr.into_records().filter_map(move |result| {
-        let record = result.ok()?;
-        let mut hash = std::collections::HashMap::new();
-        for (header, field) in headers.iter().zip(record.iter()) {
-            hash.insert(header.to_string(), field.to_string());
-        }
-        Some(hash)
-    });
-
-    Ok(Yield::Iter(iter))
-}
-
-fn parse_compat(
-    ruby: &Ruby,
-    rb_self: Value,
-    args: &[Value],
-) -> Result<Yield<impl Iterator<Item = Vec<String>>>, Error> {
-    if !ruby.block_given() {
-        return Ok(Yield::Enumerator(
-            rb_self.enumeratorize("for_each_compat", args),
-        ));
-    }
-
-    let (to_read, has_headers, delimiter) = parse_csv_args(args)?;
-    let (rdr, _) = setup_csv_parser(ruby, to_read, has_headers, delimiter)?;
-
-    let iter = rdr.into_records().filter_map(|result| {
-        result
-            .ok()
-            .map(|record| record.iter().map(|field| field.to_string()).collect())
-    });
-
-    Ok(Yield::Iter(iter))
-}
-
-use std::io::Read;
-
-struct RubyIOReader {
-    io_obj: Value,
-}
-
-impl Read for RubyIOReader {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        let result: RString = self.io_obj.funcall("read", (buf.len(),)).map_err(|_| {
-            std::io::Error::new(std::io::ErrorKind::Other, "Failed to read from IO")
-        })?;
-
-        // Handle EOF case
-        if result.is_nil() {
-            return Ok(0);
-        }
-
-        let rust_string = result.to_string().map_err(|_| {
-            std::io::Error::new(std::io::ErrorKind::Other, "Failed to convert to string")
-        })?;
-        let bytes = rust_string.as_bytes();
-
-        let bytes_to_copy = rust_string.len().min(buf.len());
-        buf[..bytes_to_copy].copy_from_slice(&bytes[..bytes_to_copy]);
-
-        Ok(bytes_to_copy)
-    }
-}
-
-impl RubyIOReader {
-    fn new(ruby: &Ruby, value: Value) -> Result<Self, Error> {
-        if value.is_kind_of(ruby.class_io()) {
-            Ok(RubyIOReader { io_obj: value })
-        } else {
-            Err(Error::new(
-                ruby.exception_runtime_error(),
-                "IO object is not a valid IO object",
-            ))
-        }
-    }
-}

data/ext/osv/src/reader.rs

@@ -0,0 +1,230 @@
+use crate::utils::*;
+use magnus::{
+    block::Yield, rb_sys::AsRawValue, value::ReprValue, Error, RClass, RString, Ruby, Value,
+};
+use std::{fs::File, io::Read, os::fd::FromRawFd, thread};
+
+/// Parses CSV data from a file and yields each row as a hash to the block.
+pub fn parse_csv(
+    ruby: &Ruby,
+    rb_self: Value,
+    args: &[Value],
+) -> Result<Yield<impl Iterator<Item = std::collections::HashMap<String, String>>>, Error> {
+    if !ruby.block_given() {
+        return Ok(Yield::Enumerator(rb_self.enumeratorize("for_each", args)));
+    }
+    let (to_read, has_headers, delimiter) = parse_csv_args(args)?;
+
+    let iter = RecordReader::<std::collections::HashMap<String, String>>::new(
+        ruby,
+        to_read,
+        has_headers,
+        delimiter.unwrap_or_else(|| ",".to_string()).as_bytes()[0],
+        1000,
+    )?;
+
+    Ok(Yield::Iter(iter))
+}
+
+pub fn parse_compat(
+    ruby: &Ruby,
+    rb_self: Value,
+    args: &[Value],
+) -> Result<Yield<impl Iterator<Item = Vec<String>>>, Error> {
+    if !ruby.block_given() {
+        return Ok(Yield::Enumerator(
+            rb_self.enumeratorize("for_each_compat", args),
+        ));
+    }
+    let (to_read, has_headers, delimiter) = parse_csv_args(args)?;
+
+    let iter = RecordReader::<Vec<String>>::new(
+        ruby,
+        to_read,
+        has_headers,
+        delimiter.unwrap_or_else(|| ",".to_string()).as_bytes()[0],
+        1000,
+    )?;
+
+    Ok(Yield::Iter(iter))
+}
+
+pub trait RecordParser {
+    type Output;
+
+    fn parse(headers: &[String], record: &csv::StringRecord) -> Self::Output;
+}
+
+impl RecordParser for std::collections::HashMap<String, String> {
+    type Output = Self;
+
+    fn parse(headers: &[String], record: &csv::StringRecord) -> Self::Output {
+        record
+            .iter()
+            .enumerate()
+            .map(|(i, field)| (headers[i].clone(), field.to_string()))
+            .collect()
+    }
+}
+
+impl RecordParser for Vec<String> {
+    type Output = Self;
+
+    fn parse(_headers: &[String], record: &csv::StringRecord) -> Self::Output {
+        record.iter().map(|field| field.to_string()).collect()
+    }
+}
+
+struct RecordReader<T: RecordParser> {
+    reader: ReadImpl<T>,
+}
+
+#[allow(dead_code)]
+enum ReadImpl<T: RecordParser> {
+    SingleThreaded {
+        reader: csv::Reader<Box<dyn Read + Send + 'static>>,
+        headers: Vec<String>,
+    },
+    MultiThreaded {
+        receiver: kanal::Receiver<T::Output>,
+        handle: Option<thread::JoinHandle<()>>,
+    },
+}
+
+impl<T: RecordParser + Send + 'static> RecordReader<T> {
+    fn new(
+        ruby: &Ruby,
+        to_read: Value,
+        has_headers: bool,
+        delimiter: u8,
+        buffer: usize,
+    ) -> Result<Self, Error> {
+        let string_io = RClass::from(ruby.eval("StringIO").map_err(|e| {
+            Error::new(
+                ruby.exception_runtime_error(),
+                format!("Failed to get StringIO class: {}", e),
+            )
+        })?);
+
+        let readable: Box<dyn Read + Send + 'static> = if to_read.is_kind_of(string_io) {
+            let string: RString = to_read.funcall("string", ()).map_err(|e| {
+                Error::new(
+                    ruby.exception_runtime_error(),
+                    format!("Failed to get string from StringIO: {}", e),
+                )
+            })?;
+            let content = string.to_string().map_err(|e| {
+                Error::new(
+                    ruby.exception_runtime_error(),
+                    format!("Failed to convert string to Rust String: {}", e),
+                )
+            })?;
+            Box::new(std::io::Cursor::new(content))
+        } else if to_read.is_kind_of(ruby.class_io()) {
+            let fd = unsafe { rb_sys::rb_io_descriptor(to_read.as_raw()) };
+            let file = unsafe { File::from_raw_fd(fd) };
+            Box::new(file)
+        } else {
+            let path = to_read
+                .to_r_string()
+                .map_err(|e| {
+                    Error::new(
+                        ruby.exception_runtime_error(),
+                        format!("Failed to convert path to string: {}", e),
+                    )
+                })?
+                .to_string()
+                .map_err(|e| {
+                    Error::new(
+                        ruby.exception_runtime_error(),
+                        format!("Failed to convert RString to Rust String: {}", e),
+                    )
+                })?;
+            let file = std::fs::File::open(&path).map_err(|e| {
+                Error::new(
+                    ruby.exception_runtime_error(),
+                    format!("Failed to open file: {}", e),
+                )
+            })?;
+            Box::new(file)
+        };
+
+        let mut reader = csv::ReaderBuilder::new()
+            .has_headers(has_headers)
+            .delimiter(delimiter)
+            .from_reader(readable);
+
+        let headers = Self::get_headers(&mut reader, has_headers)?;
+        let headers_clone = headers.clone();
+
+        let (sender, receiver) = kanal::bounded(buffer);
+        let handle = thread::spawn(move || {
+            let mut record = csv::StringRecord::new();
+            while let Ok(read) = reader.read_record(&mut record) {
+                if !read {
+                    let file_to_forget = reader.into_inner();
+                    std::mem::forget(file_to_forget);
+                    break;
+                }
+                let row = T::parse(&headers_clone, &record);
+                if sender.send(row).is_err() {
+                    break;
+                }
+            }
+        });
+
+        let read_impl = ReadImpl::MultiThreaded {
+            receiver,
+            handle: Some(handle),
+        };
+
+        Ok(Self { reader: read_impl })
+    }
+
+    fn get_headers(
+        reader: &mut csv::Reader<impl Read>,
+        has_headers: bool,
+    ) -> Result<Vec<String>, Error> {
+        let first_row = reader
+            .headers()
+            .map_err(|e| {
+                Error::new(
+                    magnus::exception::runtime_error(),
+                    format!("Failed to read headers: {}", e),
+                )
+            })?
+            .clone();
+        let num_fields = first_row.len();
+
+        Ok(if has_headers {
+            first_row.iter().map(|h| h.to_string()).collect()
+        } else {
+            (0..num_fields).map(|i| format!("c{}", i)).collect()
+        })
+    }
+}
+
+impl<T: RecordParser> Iterator for RecordReader<T> {
+    type Item = T::Output;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match &mut self.reader {
+            ReadImpl::MultiThreaded { receiver, handle } => match receiver.recv() {
+                Ok(record) => Some(record),
+                Err(_) => {
+                    if let Some(handle) = handle.take() {
+                        let _ = handle.join();
+                    }
+                    None
+                }
+            },
+            ReadImpl::SingleThreaded { reader, headers } => {
+                let mut record = csv::StringRecord::new();
+                match reader.read_record(&mut record) {
+                    Ok(true) => Some(T::parse(headers, &record)),
+                    _ => None,
+                }
+            }
+        }
+    }
+}

data/ext/osv/src/utils.rs

@@ -0,0 +1,20 @@
+use magnus::{
+    scan_args::{get_kwargs, scan_args},
+    Error, Value,
+};
+
+/// Parse common arguments for CSV parsing
+pub fn parse_csv_args(args: &[Value]) -> Result<(Value, bool, Option<String>), Error> {
+    let parsed_args = scan_args::<(Value,), (), (), (), _, ()>(args)?;
+    let (to_read,) = parsed_args.required;
+
+    let kwargs = get_kwargs::<_, (), (Option<bool>, Option<String>), ()>(
+        parsed_args.keywords,
+        &[],
+        &["has_headers", "delimiter"],
+    )?;
+
+    let has_headers = kwargs.optional.0.unwrap_or(true);
+
+    Ok((to_read, has_headers, kwargs.optional.1))
+}

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/osv.rbi

@@ -0,0 +1,29 @@
+# typed: strict
+
+module OSV
+  sig do
+    type_parameters(:T)
+      .params(
+        input: T.any(String, StringIO, IO),
+        has_headers: T.nilable(T::Boolean),
+        delimiter: T.nilable(String),
+        blk: T.proc.params(row: T::Hash[String, String]).void
+      )
+      .returns(T.untyped)
+  end
+  def self.for_each(input, has_headers: true, delimiter: nil, &blk)
+  end
+
+  sig do
+    type_parameters(:T)
+      .params(
+        input: T.any(String, StringIO, IO),
+        has_headers: T.nilable(T::Boolean),
+        delimiter: T.nilable(String),
+        blk: T.proc.params(row: T::Array[String]).void
+      )
+      .returns(T.untyped)
+  end
+  def self.for_each_compat(input, has_headers: true, delimiter: nil, &blk)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-21 00:00:00.000000000 Z
+date: 2024-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -54,7 +54,10 @@ files:
 - ext/osv/Cargo.toml
 - ext/osv/extconf.rb
 - ext/osv/src/lib.rs
+- ext/osv/src/reader.rs
+- ext/osv/src/utils.rs
 - lib/osv.rb
+- lib/osv.rbi
 - lib/osv/version.rb
 homepage: https://github.com/njaremko/osv
 licenses: