osv 0.3.12 → 0.3.14

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.14
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-24 00:00:00.000000000 Z
+date: 2025-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -38,224 +38,12 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
-description: |
-  # OSV
-
-  [![Gem Version](https://badge.fury.io/rb/osv.svg)](https://badge.fury.io/rb/osv)
-
-  OSV is a high-performance CSV parser for Ruby, implemented in Rust. It wraps BurntSushi's excellent [csv-rs](https://github.com/BurntSushi/rust-csv) crate.
-
-  It provides a simple interface for reading CSV files with support for both hash-based and array-based row formats.
-
-  The array-based mode is faster than the hash-based mode, so if you don't need the hash keys, use the array-based mode.
-
-  ## Installation
-
-  Add this line to your application's Gemfile:
-
-  ```ruby
-  gem 'osv'
-  ```
-
-  And then execute:
-
-  ```bash
-  bundle install
-  ```
-
-  Or install it directly:
-
-  ```bash
-  gem install osv
-  ```
-
-  ## Usage
-
-  ### Reading CSV Files
-
-  ```ruby
-  require 'osv'
-
-  # Basic usage - each row as a hash
-  OSV.for_each("data.csv") do |row|
-    puts row["name"]  # => "John"
-    puts row["age"]   # => "25"
-  end
-
-  # Return an enumerator instead of using a block
-  rows = OSV.for_each("data.csv")
-  rows.each { |row| puts row["name"] }
-
-  # High-performance array mode
-  OSV.for_each("data.csv", result_type: :array) do |row|
-    puts row[0]  # First column
-    puts row[1]  # Second column
-  end
-  ```
-
-  ### Input Sources
-
-  ```ruby
-  # From a file path
-  OSV.for_each("data.csv") { |row| puts row["name"] }
-
-  # From a file path
-  OSV.for_each("data.csv.gz") { |row| puts row["name"] }
-
-  # From an IO object
-  File.open("data.csv") { |file| OSV.for_each(file) { |row| puts row["name"] } }
-
-  # From a string
-  data = StringIO.new("name,age\nJohn,25")
-  OSV.for_each(data) { |row| puts row["name"] }
-  ```
-
-  ### Configuration Options
-
-  ```ruby
-  OSV.for_each("data.csv",
-    # Input formatting
-    has_headers: true,     # First row contains headers (default: true)
-    col_sep: ",",          # Column separator (default: ",")
-    quote_char: '"',       # Quote character (default: '"')
-
-    # Output formatting
-    result_type: :hash,    # :hash or :array (hash is default)
-    nil_string: nil,       # String to interpret as nil when parsing (default: nil)
-
-    # Parsing behavior
-    flexible: false,       # Allow varying number of fields (default: false)
-    flexible_default: nil, # Default value for missing fields. If unset, we ignore missing fields.
-                           # Implicitly enables flexible mode if set.
-    trim: :all,            # Whether to trim whitespace. Options are :all, :headers, or :fields (default: nil)
-    buffer_size: 1024,     # Number of rows to buffer in memory (default: 1024)
-  )
-  ```
-
-  #### Available Options
-
-  - `has_headers`: Boolean indicating if the first row contains headers (default: true)
-  - `col_sep`: String specifying the field separator (default: ",")
-  - `quote_char`: String specifying the quote character (default: "\"")
-  - `nil_string`: String that should be interpreted as nil
-    - by default, empty strings are interpreted as empty strings
-    - if you want to interpret empty strings as nil, set this to an empty string
-  - `buffer_size`: Integer specifying the number of rows to buffer in memory (default: 1024)
-  - `result_type`: String specifying the output format ("hash" or "array" or :hash or :array)
-  - `flexible`: Boolean specifying if the parser should be flexible (default: false)
-  - `flexible_default`: String specifying the default value for missing fields. Implicitly enables flexible mode if set. (default: `nil`)
-  - `trim`: String specifying the trim mode ("all" or "headers" or "fields" or :all or :headers or :fields)
-
-  When `has_headers` is false, hash keys will be generated as `"c0"`, `"c1"`, etc.
-
-  ## Requirements
-
-  - Ruby >= 3.1.0
-  - Rust toolchain (for installation from source)
-
-  ## Performance
-
-  This library is faster than the standard Ruby CSV library, and is comparable to the fastest CSV parser gems I've used.
-
-  Here's some unscientific benchmarks. You can find the code in the [benchmark/comparison_benchmark.rb](benchmark/comparison_benchmark.rb) file.
-
-  ### 10,000 lines
-
-  ```
-  Benchmarking with 100001 lines of data
-
-  ruby 3.3.6 (2024-11-05 revision 75015d4c1f) [arm64-darwin24]
-  Warming up --------------------------------------
-     OSV - Hash output     1.000 i/100ms
-     CSV - Hash output     1.000 i/100ms
-    OSV - Array output     1.000 i/100ms
-  OSV - Direct Open Array output
-                          12.719M i/100ms
-    CSV - Array output     1.000 i/100ms
-  FastCSV - Array output
-                           1.000 i/100ms
-        OSV - StringIO     1.000 i/100ms
-        CSV - StringIO     1.000 i/100ms
-    FastCSV - StringIO     1.000 i/100ms
-         OSV - Gzipped     1.000 i/100ms
-         CSV - Gzipped     1.000 i/100ms
-  Calculating -------------------------------------
-     OSV - Hash output      6.722 (±14.9%) i/s  (148.77 ms/i) -     59.000 in  10.074753s
-     CSV - Hash output      1.223 (± 0.0%) i/s  (817.62 ms/i) -     13.000 in  10.788284s
-    OSV - Array output     17.284 (±11.6%) i/s   (57.86 ms/i) -    171.000 in  10.007321s
-  OSV - Direct Open Array output
-                          213.629M (±13.5%) i/s    (4.68 ns/i) -      1.921B in  10.005506s
-    CSV - Array output      2.193 (± 0.0%) i/s  (455.93 ms/i) -     22.000 in  10.052607s
-  FastCSV - Array output
-                            7.993 (± 0.0%) i/s  (125.11 ms/i) -     80.000 in  10.053729s
-        OSV - StringIO      6.626 (±15.1%) i/s  (150.91 ms/i) -     66.000 in  10.103646s
-        CSV - StringIO      1.478 (± 0.0%) i/s  (676.78 ms/i) -     15.000 in  10.158640s
-    FastCSV - StringIO     17.074 (± 5.9%) i/s   (58.57 ms/i) -    171.000 in  10.059266s
-         OSV - Gzipped      5.639 (± 0.0%) i/s  (177.32 ms/i) -     57.000 in  10.152487s
-         CSV - Gzipped      1.176 (± 0.0%) i/s  (850.19 ms/i) -     12.000 in  10.233398s
-
-  Comparison:
-  OSV - Direct Open Array output: 213629268.6 i/s
-    OSV - Array output:       17.3 i/s - 12360250.79x  slower
-    FastCSV - StringIO:       17.1 i/s - 12511956.50x  slower
-  FastCSV - Array output:        8.0 i/s - 26727225.72x  slower
-     OSV - Hash output:        6.7 i/s - 31780615.83x  slower
-        OSV - StringIO:        6.6 i/s - 32239620.60x  slower
-         OSV - Gzipped:        5.6 i/s - 37881517.48x  slower
-    CSV - Array output:        2.2 i/s - 97400427.87x  slower
-        CSV - StringIO:        1.5 i/s - 144580048.04x  slower
-     CSV - Hash output:        1.2 i/s - 174666591.31x  slower
-         CSV - Gzipped:        1.2 i/s - 181626018.23x  slower
-  ```
-
-  ### 1,000,000 lines
-
-  ```
-  Benchmarking with 1000001 lines of data
-
-  ruby 3.3.6 (2024-11-05 revision 75015d4c1f) [arm64-darwin24]
-  Warming up --------------------------------------
-     OSV - Hash output     1.000 i/100ms
-     CSV - Hash output     1.000 i/100ms
-    OSV - Array output     1.000 i/100ms
-  OSV - Direct Open Array output
-                           1.000 i/100ms
-    CSV - Array output     1.000 i/100ms
-  FastCSV - Array output
-                           1.000 i/100ms
-        OSV - StringIO     1.000 i/100ms
-        CSV - StringIO     1.000 i/100ms
-    FastCSV - StringIO     1.000 i/100ms
-         OSV - Gzipped     1.000 i/100ms
-         CSV - Gzipped     1.000 i/100ms
-  Calculating -------------------------------------
-     OSV - Hash output      0.492 (± 0.0%) i/s     (2.03 s/i) -      5.000 in  10.463278s
-     CSV - Hash output      0.114 (± 0.0%) i/s     (8.75 s/i) -      2.000 in  17.573877s
-    OSV - Array output      1.502 (± 0.0%) i/s  (665.58 ms/i) -     14.000 in  10.217551s
-  OSV - Direct Open Array output
-                            1.626 (± 0.0%) i/s  (614.90 ms/i) -     16.000 in  10.190323s
-    CSV - Array output      0.183 (± 0.0%) i/s     (5.46 s/i) -      2.000 in  10.951943s
-  FastCSV - Array output
-                            0.326 (± 0.0%) i/s     (3.07 s/i) -      4.000 in  12.340605s
-        OSV - StringIO      0.567 (± 0.0%) i/s     (1.76 s/i) -      6.000 in  10.698027s
-        CSV - StringIO      0.141 (± 0.0%) i/s     (7.10 s/i) -      2.000 in  14.237144s
-    FastCSV - StringIO      0.923 (± 0.0%) i/s     (1.08 s/i) -     10.000 in  11.567775s
-         OSV - Gzipped      0.437 (± 0.0%) i/s     (2.29 s/i) -      5.000 in  11.452764s
-         CSV - Gzipped      0.104 (± 0.0%) i/s     (9.64 s/i) -      2.000 in  19.373423s
-
-  Comparison:
-  OSV - Direct Open Array output:        1.6 i/s
-    OSV - Array output:        1.5 i/s - 1.08x  slower
-    FastCSV - StringIO:        0.9 i/s - 1.76x  slower
-        OSV - StringIO:        0.6 i/s - 2.87x  slower
-     OSV - Hash output:        0.5 i/s - 3.30x  slower
-         OSV - Gzipped:        0.4 i/s - 3.72x  slower
-  FastCSV - Array output:        0.3 i/s - 4.99x  slower
-    CSV - Array output:        0.2 i/s - 8.88x  slower
-        CSV - StringIO:        0.1 i/s - 11.55x  slower
-     CSV - Hash output:        0.1 i/s - 14.24x  slower
-         CSV - Gzipped:        0.1 i/s - 15.68x  slower
-  ```
+description: |2
+      OSV is a high-performance CSV parser for Ruby, implemented in Rust.
+      It wraps BurntSushi's csv-rs crate to provide fast CSV parsing with support for both hash-based and array-based row formats.
+      Features include: Flexible input sources (file paths, gzipped files, IO objects, strings),
+      configurable parsing options (headers, separators, quote chars), support for both hash and array output formats,
+      whitespace trimming options, strict or flexible parsing modes, and is significantly faster than Ruby's standard CSV library.
 email:
 - nathan@jaremko.ca
 executables: []
@@ -275,9 +63,9 @@ files:
 - ext/osv/src/csv/header_cache.rs
 - ext/osv/src/csv/mod.rs
 - ext/osv/src/csv/parser.rs
-- ext/osv/src/csv/read_impl.rs
-- ext/osv/src/csv/reader.rs
 - ext/osv/src/csv/record.rs
+- ext/osv/src/csv/record_reader.rs
+- ext/osv/src/csv/ruby_reader.rs
 - ext/osv/src/lib.rs
 - ext/osv/src/reader.rs
 - ext/osv/src/utils.rs
@@ -290,6 +78,10 @@ licenses:
 metadata:
   homepage_uri: https://github.com/njaremko/osv
   source_code_uri: https://github.com/njaremko/osv
+  readme_uri: https://github.com/njaremko/osv/blob/main/README.md
+  changelog_uri: https://github.com/njaremko/osv/blob/main/CHANGELOG.md
+  documentation_uri: https://www.rubydoc.info/gems/osv
+  funding_uri: https://github.com/sponsors/njaremko
 post_install_message:
 rdoc_options: []
 require_paths:

data/ext/osv/src/csv/read_impl.rs

@@ -1,75 +0,0 @@
-use super::{header_cache::StringCache, parser::RecordParser};
-use std::{io::Read, thread};
-
-pub(crate) const READ_BUFFER_SIZE: usize = 8192;
-
-pub enum ReadImpl<T: RecordParser> {
-    SingleThreaded {
-        reader: csv::Reader<Box<dyn Read>>,
-        headers: Vec<&'static str>,
-        null_string: Option<String>,
-        flexible_default: Option<String>,
-    },
-    MultiThreaded {
-        headers: Vec<&'static str>,
-        receiver: kanal::Receiver<T::Output>,
-        handle: Option<thread::JoinHandle<()>>,
-    },
-}
-
-impl<T: RecordParser> ReadImpl<T> {
-    #[inline]
-    pub fn next(&mut self) -> Option<T::Output> {
-        match self {
-            Self::MultiThreaded {
-                receiver, handle, ..
-            } => match receiver.recv() {
-                Ok(record) => Some(record),
-                Err(_) => {
-                    if let Some(handle) = handle.take() {
-                        let _ = handle.join();
-                    }
-                    None
-                }
-            },
-            Self::SingleThreaded {
-                reader,
-                headers,
-                null_string,
-                flexible_default,
-            } => {
-                let mut record = csv::StringRecord::with_capacity(READ_BUFFER_SIZE, headers.len());
-                match reader.read_record(&mut record) {
-                    Ok(true) => Some(T::parse(
-                        headers,
-                        &record,
-                        null_string.as_deref(),
-                        flexible_default.as_deref(),
-                    )),
-                    _ => None,
-                }
-            }
-        }
-    }
-
-    #[inline]
-    pub fn cleanup(&mut self) {
-        match self {
-            Self::MultiThreaded {
-                receiver,
-                handle,
-                headers,
-                ..
-            } => {
-                receiver.close();
-                if let Some(handle) = handle.take() {
-                    let _ = handle.join();
-                }
-                let _ = StringCache::clear(headers);
-            }
-            Self::SingleThreaded { headers, .. } => {
-                let _ = StringCache::clear(headers);
-            }
-        }
-    }
-}

data/ext/osv/src/csv/reader.rs

@@ -1,57 +0,0 @@
-use super::{parser::RecordParser, read_impl::ReadImpl};
-use magnus::{Error, Ruby};
-use std::{borrow::Cow, io::Read};
-
-pub struct RecordReader<T: RecordParser> {
-    pub(crate) reader: ReadImpl<T>,
-}
-
-impl<T: RecordParser> RecordReader<T> {
-    #[inline]
-    pub(crate) fn get_headers(
-        ruby: &Ruby,
-        reader: &mut csv::Reader<impl Read>,
-        has_headers: bool,
-    ) -> Result<Vec<String>, Error> {
-        let first_row = reader.headers().map_err(|e| {
-            Error::new(
-                ruby.exception_runtime_error(),
-                Cow::Owned(format!("Failed to read headers: {e}")),
-            )
-        })?;
-
-        Ok(if has_headers {
-            // Pre-allocate the vector with exact capacity
-            let mut headers = Vec::with_capacity(first_row.len());
-            headers.extend(first_row.iter().map(String::from));
-            headers
-        } else {
-            // Pre-allocate the vector with exact capacity
-            let mut headers = Vec::with_capacity(first_row.len());
-            headers.extend((0..first_row.len()).map(|i| format!("c{i}")));
-            headers
-        })
-    }
-}
-
-impl<T: RecordParser> Iterator for RecordReader<T> {
-    type Item = T::Output;
-
-    #[inline]
-    fn next(&mut self) -> Option<Self::Item> {
-        self.reader.next()
-    }
-
-    #[inline]
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        // We can't know the exact size without reading the whole file
-        (0, None)
-    }
-}
-
-impl<T: RecordParser> Drop for RecordReader<T> {
-    #[inline]
-    fn drop(&mut self) {
-        self.reader.cleanup();
-    }
-}