osv 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51e4a387f1ed43bddc9f1f7a118637953d04239b5324ef131b9c860577ed4d41
-  data.tar.gz: e42928a09656216bbadcc2458953a8c5f28401ddf27095fc05038e0960471854
+  metadata.gz: 137ae556685639f7d13234e3061d9b310757ce02f75a713753d175f1bc71b628
+  data.tar.gz: 5892494ad08d783955d2b932150d65433a4d3593376fadbaf54e54780e7a350f
 SHA512:
-  metadata.gz: 4100c50a629ba5803db883532cfbe547eb3091e421b0876595d91791d8952a7b0169477c9c6f31063eafa5b91d0a9b1a9f0a5ae016d70cdd101e284beebfaf22
-  data.tar.gz: 90a822c644fcb37dc1892ede85a54395bc9e62a4b0b0a1af838182d390702d0ee4253151faafcedbf734b0a381fe2acf5c1ab23b842059fbdd4d51570fe33e58
+  metadata.gz: 6efbc2ee65a8e79379722ae977ee7dbec6131b78968d080f9feb86a3310368c387da54dd8c073e9b4008cb80d906293ea9115982d00d5ff637cf5ab51179b53c
+  data.tar.gz: 7b4ab3199f90654cd831dfbb52a9d22b70237e7120bd5308a1b7698268fa981abefd7ee47d53424d0c7bff46956256db8f1e139d17e381fd5570a16ca183e376

data/ext/osv/src/csv/builder.rs

@@ -213,7 +213,8 @@ impl<'a, T: RecordParser<'a>> RecordReaderBuilder<'a, T> {
             .from_reader(reader);
 
         let mut headers =
-            RecordReader::<T>::get_headers(&self.ruby, &mut reader, self.has_headers)?;
+            RecordReader::<T>::get_headers(&self.ruby, &mut reader, self.has_headers, self.lossy)?;
+
         if self.ignore_null_bytes {
             headers = headers.iter().map(|h| h.replace("\0", "")).collect();
         }

data/ext/osv/src/csv/header_cache.rs

@@ -8,7 +8,7 @@ use std::{
     collections::HashMap,
     sync::{
         atomic::{AtomicU32, Ordering},
-        LazyLock, Mutex, OnceLock,
+        Arc, LazyLock, Mutex, OnceLock,
     },
 };
 
@@ -22,12 +22,11 @@ pub enum CacheError {
     LockError(String),
 }
 
-static STRING_CACHE: LazyLock<Mutex<HashMap<&'static str, (StringCacheKey, AtomicU32)>>> =
+static STRING_CACHE: LazyLock<Mutex<HashMap<&'static str, (Arc<StringCacheKey>, AtomicU32)>>> =
     LazyLock::new(|| Mutex::new(HashMap::with_capacity(100)));
 
 pub struct StringCache;
 
-#[derive(Copy, Clone)]
 pub struct StringCacheKey(Opaque<FString>, &'static str);
 
 impl StringCacheKey {
@@ -50,6 +49,12 @@ impl IntoValue for StringCacheKey {
     }
 }
 
+impl IntoValue for &StringCacheKey {
+    fn into_value_with(self, handle: &Ruby) -> Value {
+        handle.into_value(self.0)
+    }
+}
+
 impl std::fmt::Debug for StringCacheKey {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         self.1.fmt(f)
@@ -72,43 +77,43 @@ impl std::hash::Hash for StringCacheKey {
 
 impl StringCache {
     #[allow(dead_code)]
-    pub fn intern(string: String) -> Result<StringCacheKey, CacheError> {
+    pub fn intern(string: String) -> Result<Arc<StringCacheKey>, CacheError> {
         let mut cache = STRING_CACHE
             .lock()
             .map_err(|e| CacheError::LockError(e.to_string()))?;
 
         if let Some((_, (interned_string, counter))) = cache.get_key_value(string.as_str()) {
             counter.fetch_add(1, Ordering::Relaxed);
-            Ok(*interned_string)
+            Ok(interned_string.clone())
         } else {
-            let interned = StringCacheKey::new(string.as_str());
+            let interned = Arc::new(StringCacheKey::new(string.as_str()));
             let leaked = Box::leak(string.into_boxed_str());
-            cache.insert(leaked, (interned, AtomicU32::new(1)));
+            cache.insert(leaked, (interned.clone(), AtomicU32::new(1)));
             Ok(interned)
         }
     }
 
-    pub fn intern_many(strings: &[String]) -> Result<Vec<StringCacheKey>, CacheError> {
+    pub fn intern_many(strings: &[String]) -> Result<Vec<Arc<StringCacheKey>>, CacheError> {
         let mut cache = STRING_CACHE
             .lock()
             .map_err(|e| CacheError::LockError(e.to_string()))?;
 
-        let mut result: Vec<StringCacheKey> = Vec::with_capacity(strings.len());
+        let mut result: Vec<Arc<StringCacheKey>> = Vec::with_capacity(strings.len());
         for string in strings {
             if let Some((_, (interned_string, counter))) = cache.get_key_value(string.as_str()) {
                 counter.fetch_add(1, Ordering::Relaxed);
-                result.push(*interned_string);
+                result.push(interned_string.clone());
             } else {
-                let interned = StringCacheKey::new(string);
+                let interned = Arc::new(StringCacheKey::new(string));
                 let leaked = Box::leak(string.clone().into_boxed_str());
-                cache.insert(leaked, (interned, AtomicU32::new(1)));
+                cache.insert(leaked, (interned.clone(), AtomicU32::new(1)));
                 result.push(interned);
             }
         }
         Ok(result)
     }
 
-    pub fn clear(headers: &[StringCacheKey]) -> Result<(), CacheError> {
+    pub fn clear(headers: &[Arc<StringCacheKey>]) -> Result<(), CacheError> {
         let mut cache = STRING_CACHE
             .lock()
             .map_err(|e| CacheError::LockError(e.to_string()))?;
@@ -116,7 +121,7 @@ impl StringCache {
         let to_remove: Vec<_> = headers
             .iter()
             .filter_map(|header| {
-                let key = header.as_ref();
+                let key = header.as_ref().as_ref();
                 if let Some((_, (_, counter))) = cache.get_key_value(key) {
                     let prev_count = counter.fetch_sub(1, Ordering::Relaxed);
                     if prev_count == 1 {
@@ -140,7 +145,7 @@ impl StringCache {
 
 pub struct HeaderCacheCleanupIter<I> {
     pub inner: I,
-    pub headers: OnceLock<Vec<StringCacheKey>>,
+    pub headers: OnceLock<Vec<Arc<StringCacheKey>>>,
 }
 
 impl<I: Iterator> Iterator for HeaderCacheCleanupIter<I> {

data/ext/osv/src/csv/parser.rs

@@ -1,6 +1,7 @@
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::hash::BuildHasher;
+use std::sync::Arc;
 
 use super::header_cache::StringCacheKey;
 use super::CowStr;
@@ -14,7 +15,7 @@ pub trait RecordParser<'a> {
     type Output;
 
     fn parse(
-        headers: &[StringCacheKey],
+        headers: &[Arc<StringCacheKey>],
         record: &CsvRecordType,
         null_string: Option<Cow<'a, str>>,
         ignore_null_bytes: bool,
@@ -22,13 +23,13 @@ pub trait RecordParser<'a> {
 }
 
 impl<'a, S: BuildHasher + Default> RecordParser<'a>
-    for HashMap<StringCacheKey, Option<CowStr<'a>>, S>
+    for HashMap<Arc<StringCacheKey>, Option<CowStr<'a>>, S>
 {
     type Output = Self;
 
     #[inline]
     fn parse(
-        headers: &[StringCacheKey],
+        headers: &[Arc<StringCacheKey>],
         record: &CsvRecordType,
         null_string: Option<Cow<'a, str>>,
         ignore_null_bytes: bool,
@@ -65,7 +66,7 @@ impl<'a, S: BuildHasher + Default> RecordParser<'a>
                 }),
             };
 
-            map.insert(*header, value);
+            map.insert(header.clone(), value);
         });
         map
     }
@@ -76,7 +77,7 @@ impl<'a> RecordParser<'a> for Vec<Option<CowStr<'a>>> {
 
     #[inline]
     fn parse(
-        headers: &[StringCacheKey],
+        headers: &[Arc<StringCacheKey>],
         record: &CsvRecordType,
         null_string: Option<Cow<'a, str>>,
         ignore_null_bytes: bool,

data/ext/osv/src/csv/record.rs

@@ -1,13 +1,13 @@
 use itertools::Itertools;
 use magnus::{value::ReprValue, IntoValue, Ruby, Value};
-use std::{borrow::Cow, collections::HashMap, hash::BuildHasher};
+use std::{borrow::Cow, collections::HashMap, hash::BuildHasher, sync::Arc};
 
 use super::StringCacheKey;
 
 #[derive(Debug)]
 pub enum CsvRecord<'a, S: BuildHasher + Default> {
     Vec(Vec<Option<CowStr<'a>>>),
-    Map(HashMap<StringCacheKey, Option<CowStr<'a>>, S>),
+    Map(HashMap<Arc<StringCacheKey>, Option<CowStr<'a>>, S>),
 }
 
 impl<S: BuildHasher + Default> IntoValue for CsvRecord<'_, S> {
@@ -28,7 +28,7 @@ impl<S: BuildHasher + Default> IntoValue for CsvRecord<'_, S> {
 
                 for chunk in &map.into_iter().chunks(128) {
                     for (k, v) in chunk {
-                        values[i] = handle.into_value(k);
+                        values[i] = handle.into_value(k.as_ref());
                         values[i + 1] = handle.into_value(v);
                         i += 2;
                     }

data/ext/osv/src/csv/record_reader.rs

@@ -5,6 +5,7 @@ use super::{header_cache::StringCache, ruby_reader::SeekableRead};
 use magnus::{Error, Ruby};
 use std::borrow::Cow;
 use std::io::{BufReader, Read};
+use std::sync::Arc;
 
 /// Size of the internal buffer used for reading CSV records
 pub(crate) const READ_BUFFER_SIZE: usize = 16384;
@@ -14,7 +15,7 @@ pub(crate) const READ_BUFFER_SIZE: usize = 16384;
 /// This struct implements Iterator to provide a streaming interface for CSV records.
 pub struct RecordReader<'a, T: RecordParser<'a>> {
     reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
-    headers: Vec<StringCacheKey>,
+    headers: Vec<Arc<StringCacheKey>>,
     null_string: Option<Cow<'a, str>>,
     string_record: CsvRecordType,
     parser: std::marker::PhantomData<T>,
@@ -36,25 +37,45 @@ impl<'a, T: RecordParser<'a>> RecordReader<'a, T> {
         ruby: &Ruby,
         reader: &mut csv::Reader<impl Read>,
         has_headers: bool,
+        lossy: bool,
     ) -> Result<Vec<String>, Error> {
-        let first_row = reader.headers().map_err(|e| {
-            Error::new(
-                ruby.exception_runtime_error(),
-                format!("Failed to read headers: {e}"),
-            )
-        })?;
-
-        Ok(if has_headers {
-            first_row.iter().map(String::from).collect()
+        let headers = if lossy {
+            let first_row = reader.byte_headers().map_err(|e| {
+                Error::new(
+                    ruby.exception_runtime_error(),
+                    format!("Failed to read headers: {e}"),
+                )
+            })?;
+            if has_headers {
+                first_row
+                    .iter()
+                    .map(String::from_utf8_lossy)
+                    .map(|x| x.to_string())
+                    .collect()
+            } else {
+                (0..first_row.len()).map(|i| format!("c{i}")).collect()
+            }
         } else {
-            (0..first_row.len()).map(|i| format!("c{i}")).collect()
-        })
+            let first_row = reader.headers().map_err(|e| {
+                Error::new(
+                    ruby.exception_runtime_error(),
+                    format!("Failed to read headers: {e}"),
+                )
+            })?;
+            if has_headers {
+                first_row.iter().map(String::from).collect()
+            } else {
+                (0..first_row.len()).map(|i| format!("c{i}")).collect()
+            }
+        };
+
+        Ok(headers)
     }
 
     /// Creates a new RecordReader instance.
     pub(crate) fn new(
         reader: csv::Reader<BufReader<Box<dyn SeekableRead>>>,
-        headers: Vec<StringCacheKey>,
+        headers: Vec<Arc<StringCacheKey>>,
         null_string: Option<Cow<'a, str>>,
         ignore_null_bytes: bool,
         lossy: bool,

data/ext/osv/src/reader.rs

@@ -5,6 +5,7 @@ use csv::Trim;
 use magnus::value::ReprValue;
 use magnus::{Error, IntoValue, KwArgs, RHash, Ruby, Symbol, Value};
 use std::collections::HashMap;
+use std::sync::Arc;
 
 /// Valid result types for CSV parsing
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -93,7 +94,7 @@ pub fn parse_csv(rb_self: Value, args: &[Value]) -> Result<Value, Error> {
     match result_type {
         ResultType::Hash => {
             let builder = RecordReaderBuilder::<
-                HashMap<StringCacheKey, Option<CowStr<'static>>, RandomState>,
+                HashMap<Arc<StringCacheKey>, Option<CowStr<'static>>, RandomState>,
             >::new(ruby, to_read)
             .has_headers(has_headers)
             .flexible(flexible)

data/lib/osv/version.rb

@@ -1,3 +1,3 @@
 module OSV
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: osv
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-30 00:00:00.000000000 Z
+date: 2025-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys