parquet 0.5.12 → 0.6.0

data/ext/parquet-ruby-adapter/src/error.rs

@@ -0,0 +1,148 @@
+use magnus::{Error as MagnusError, Ruby};
+use parquet_core::ParquetError as CoreParquetError;
+use std::fmt::Display;
+use thiserror::Error;
+
+/// Error type for parquet-ruby-adapter
+#[derive(Error, Debug)]
+pub enum RubyAdapterError {
+    /// Core parquet errors
+    #[error("Parquet error: {0}")]
+    Parquet(#[from] CoreParquetError),
+
+    /// Magnus/Ruby errors
+    #[error("Ruby error: {0}")]
+    Ruby(String),
+
+    /// IO errors
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+
+    /// Type conversion errors
+    #[error("Type conversion error: {0}")]
+    TypeConversion(String),
+
+    /// Schema conversion errors
+    #[error("Schema conversion error: {0}")]
+    SchemaConversion(String),
+
+    /// Metadata extraction errors
+    #[error("Metadata error: {0}")]
+    Metadata(String),
+
+    /// Invalid input errors
+    #[error("Invalid input: {0}")]
+    InvalidInput(String),
+
+    /// Runtime errors
+    #[error("Runtime error: {0}")]
+    Runtime(String),
+}
+
+pub type Result<T> = std::result::Result<T, RubyAdapterError>;
+
+impl RubyAdapterError {
+    /// Create a new Ruby error
+    pub fn ruby<S: Into<String>>(msg: S) -> Self {
+        RubyAdapterError::Ruby(msg.into())
+    }
+
+    /// Create a new type conversion error
+    pub fn type_conversion<S: Into<String>>(msg: S) -> Self {
+        RubyAdapterError::TypeConversion(msg.into())
+    }
+
+    /// Create a new schema conversion error
+    pub fn schema_conversion<S: Into<String>>(msg: S) -> Self {
+        RubyAdapterError::SchemaConversion(msg.into())
+    }
+
+    /// Create a new metadata error
+    pub fn metadata<S: Into<String>>(msg: S) -> Self {
+        RubyAdapterError::Metadata(msg.into())
+    }
+
+    /// Create a new invalid input error
+    pub fn invalid_input<S: Into<String>>(msg: S) -> Self {
+        RubyAdapterError::InvalidInput(msg.into())
+    }
+
+    /// Create a new runtime error
+    pub fn runtime<S: Into<String>>(msg: S) -> Self {
+        RubyAdapterError::Runtime(msg.into())
+    }
+}
+
+/// Convert RubyAdapterError to MagnusError
+impl From<RubyAdapterError> for MagnusError {
+    fn from(err: RubyAdapterError) -> Self {
+        match Ruby::get() {
+            Ok(ruby) => match &err {
+                RubyAdapterError::Io(_) => {
+                    MagnusError::new(ruby.exception_io_error(), err.to_string())
+                }
+                RubyAdapterError::TypeConversion(_) => {
+                    MagnusError::new(ruby.exception_type_error(), err.to_string())
+                }
+                RubyAdapterError::InvalidInput(_) => {
+                    MagnusError::new(ruby.exception_arg_error(), err.to_string())
+                }
+                _ => MagnusError::new(ruby.exception_runtime_error(), err.to_string()),
+            },
+            Err(_) => {
+                // Fallback if we can't get Ruby runtime
+                MagnusError::new(
+                    magnus::exception::runtime_error(),
+                    format!("Failed to get Ruby runtime: {}", err),
+                )
+            }
+        }
+    }
+}
+
+/// Extension trait to convert errors to MagnusError at the boundary
+pub trait IntoMagnusError<T> {
+    /// Convert to MagnusError
+    fn into_magnus_error(self) -> std::result::Result<T, MagnusError>;
+}
+
+impl<T> IntoMagnusError<T> for Result<T> {
+    fn into_magnus_error(self) -> std::result::Result<T, MagnusError> {
+        self.map_err(Into::into)
+    }
+}
+
+/// Extension trait to add context to errors
+pub trait ErrorContext<T> {
+    /// Add context to an error
+    fn context<S: Display>(self, ctx: S) -> Result<T>;
+
+    /// Add context with a closure that's only called on error
+    fn with_context<S: Display, F: FnOnce() -> S>(self, f: F) -> Result<T>;
+}
+
+impl<T, E> ErrorContext<T> for std::result::Result<T, E>
+where
+    E: Into<RubyAdapterError>,
+{
+    fn context<S: Display>(self, ctx: S) -> Result<T> {
+        self.map_err(|e| {
+            let base_error = e.into();
+            RubyAdapterError::Runtime(format!("{}: {}", ctx, base_error))
+        })
+    }
+
+    fn with_context<S: Display, F: FnOnce() -> S>(self, f: F) -> Result<T> {
+        self.map_err(|e| {
+            let base_error = e.into();
+            RubyAdapterError::Runtime(format!("{}: {}", f(), base_error))
+        })
+    }
+}
+
+/// Convert from MagnusError to RubyAdapterError
+impl From<MagnusError> for RubyAdapterError {
+    fn from(err: MagnusError) -> Self {
+        RubyAdapterError::Ruby(err.to_string())
+    }
+}

data/ext/{parquet/src/ruby_reader.rs → parquet-ruby-adapter/src/io.rs}

@@ -1,23 +1,21 @@
 use bytes::Bytes;
-use magnus::{
-    value::{Opaque, ReprValue},
-    RString, Ruby, Value,
-};
+use magnus::value::{Opaque, ReprValue};
+use magnus::{Error as MagnusError, RString, Ruby, Value};
+use std::io::{Error as IoError, ErrorKind, Read, Seek, SeekFrom, Write};
+
 use parquet::{
     errors::ParquetError,
     file::reader::{ChunkReader, Length},
 };
-use std::{fs::File, rc::Rc, sync::Mutex};
+use std::{fs::File, sync::Mutex};
 use std::{
-    io::{self, BufReader, Read, Seek, SeekFrom, Write},
+    io::{self, BufReader},
     sync::Arc,
 };
 
-use crate::types::ParquetGemError;
-
 /// A reader that can handle various Ruby input types (String, StringIO, IO-like objects)
 /// and provide a standard Read implementation for them.
-pub enum RubyReader {
+pub enum RubyIOReader {
     String {
         inner: Opaque<RString>,
         offset: usize,
@@ -32,38 +30,35 @@ pub enum RubyReader {
 
 // Sending is technically not safe, but the only things that threatens to
 // do this is the parquet gem, and they don't seem to actually do it.
-unsafe impl Send for RubyReader {}
+unsafe impl Send for RubyIOReader {}
 
-impl RubyReader {
-    pub fn new(ruby: Rc<Ruby>, value: Value) -> Result<Self, ParquetGemError> {
-        if RubyReader::is_seekable_io_like(&value) {
-            Ok(RubyReader::RubyIoLike {
+impl RubyIOReader {
+    pub fn new(value: Value) -> std::io::Result<Self> {
+        if RubyIOReader::is_seekable_io_like(&value) {
+            Ok(RubyIOReader::RubyIoLike {
                 inner: Opaque::from(value),
             })
-        } else if RubyReader::is_io_like(&value) {
-            let mut temp_file = tempfile::tempfile()
-                .map_err(|e| magnus::Error::new(ruby.exception_runtime_error(), e.to_string()))?;
+        } else if RubyIOReader::is_io_like(&value) {
+            let mut temp_file = tempfile::tempfile()?;
 
             // This is safe, because we won't call seek
-            let inner_readable = RubyReader::RubyIoLike {
+            let inner_readable = RubyIOReader::RubyIoLike {
                 inner: Opaque::from(value),
             };
             let mut reader = BufReader::new(inner_readable);
-            io::copy(&mut reader, &mut temp_file)
-                .map_err(|e| magnus::Error::new(ruby.exception_runtime_error(), e.to_string()))?;
-            temp_file
-                .seek(SeekFrom::Start(0))
-                .map_err(|e| magnus::Error::new(ruby.exception_runtime_error(), e.to_string()))?;
+            io::copy(&mut reader, &mut temp_file)?;
+            temp_file.seek(SeekFrom::Start(0))?;
 
-            Ok(RubyReader::NativeProxyIoLike {
+            Ok(RubyIOReader::NativeProxyIoLike {
                 proxy_file: temp_file,
             })
         } else {
             // Try calling `to_str`, and if that fails, try `to_s`
             let string_content = value
                 .funcall::<_, _, RString>("to_str", ())
-                .or_else(|_| value.funcall::<_, _, RString>("to_s", ()))?;
-            Ok(RubyReader::String {
+                .or_else(|_| value.funcall::<_, _, RString>("to_s", ()))
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            Ok(RubyIOReader::String {
                 inner: Opaque::from(string_content),
                 offset: 0,
             })
@@ -82,12 +77,13 @@ impl RubyReader {
     }
 }
 
-impl Seek for RubyReader {
+impl Seek for RubyIOReader {
     fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
-        let ruby = unsafe { Ruby::get_unchecked() };
+        let ruby = Ruby::get()
+            .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to get Ruby runtime"))?;
         match self {
-            RubyReader::NativeProxyIoLike { proxy_file } => proxy_file.seek(pos),
-            RubyReader::String {
+            RubyIOReader::NativeProxyIoLike { proxy_file } => proxy_file.seek(pos),
+            RubyIOReader::String {
                 inner,
                 offset: original_offset,
             } => {
@@ -108,7 +104,7 @@ impl Seek for RubyReader {
                 *original_offset = new_offset.min(unwrapped_inner.len());
                 Ok(*original_offset as u64)
             }
-            RubyReader::RubyIoLike { inner } => {
+            RubyIOReader::RubyIoLike { inner } => {
                 let unwrapped_inner = ruby.get_inner(*inner);
 
                 let (whence, ruby_offset) = match pos {
@@ -131,12 +127,13 @@ impl Seek for RubyReader {
     }
 }
 
-impl Read for RubyReader {
+impl Read for RubyIOReader {
     fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {
-        let ruby = unsafe { Ruby::get_unchecked() };
+        let ruby = Ruby::get()
+            .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to get Ruby runtime"))?;
         match self {
-            RubyReader::NativeProxyIoLike { proxy_file } => proxy_file.read(buf),
-            RubyReader::String { inner, offset } => {
+            RubyIOReader::NativeProxyIoLike { proxy_file } => proxy_file.read(buf),
+            RubyIOReader::String { inner, offset } => {
                 let unwrapped_inner = ruby.get_inner(*inner);
 
                 let string_buffer = unsafe { unwrapped_inner.as_slice() };
@@ -152,7 +149,7 @@ impl Read for RubyReader {
 
                 Ok(copy_size)
             }
-            RubyReader::RubyIoLike { inner } => {
+            RubyIOReader::RubyIoLike { inner } => {
                 let unwrapped_inner = ruby.get_inner(*inner);
 
                 let bytes = unwrapped_inner
@@ -172,16 +169,22 @@ impl Read for RubyReader {
     }
 }
 
-impl Length for RubyReader {
+impl Length for RubyIOReader {
     fn len(&self) -> u64 {
-        let ruby = unsafe { Ruby::get_unchecked() };
+        let ruby = match Ruby::get() {
+            Ok(r) => r,
+            Err(_) => {
+                eprintln!("Failed to get Ruby runtime in RubyIOReader::len");
+                return 0;
+            }
+        };
         match self {
-            RubyReader::NativeProxyIoLike { proxy_file } => proxy_file.len(),
-            RubyReader::String { inner, offset: _ } => {
+            RubyIOReader::NativeProxyIoLike { proxy_file } => proxy_file.len(),
+            RubyIOReader::String { inner, offset: _ } => {
                 let unwrapped_inner = ruby.get_inner(*inner);
                 unwrapped_inner.len() as u64
             }
-            RubyReader::RubyIoLike { inner } => {
+            RubyIOReader::RubyIoLike { inner } => {
                 let unwrapped_inner = ruby.get_inner(*inner);
 
                 // Get current position
@@ -222,12 +225,13 @@ impl Length for RubyReader {
                     }
                 };
 
-                assert_eq!(
-                    current_pos, final_pos,
-                    "Failed to restore original position in seekable IO object. Started at position {}, but ended at {}",
-                    current_pos,
-                    final_pos
-                );
+                if current_pos != final_pos {
+                    eprintln!(
+                        "Failed to restore original position in seekable IO object. Started at position {}, but ended at {}",
+                        current_pos,
+                        final_pos
+                    );
+                }
 
                 size
             }
@@ -238,21 +242,28 @@ impl Length for RubyReader {
 const READ_BUFFER_SIZE: usize = 16 * 1024;
 
 #[derive(Clone)]
-pub struct ThreadSafeRubyReader(Arc<Mutex<RubyReader>>);
+pub struct ThreadSafeRubyIOReader(Arc<Mutex<RubyIOReader>>);
 
-impl ThreadSafeRubyReader {
-    pub fn new(reader: RubyReader) -> Self {
+impl ThreadSafeRubyIOReader {
+    pub fn new(reader: RubyIOReader) -> Self {
         Self(Arc::new(Mutex::new(reader)))
     }
 }
 
-impl Length for ThreadSafeRubyReader {
+impl Length for ThreadSafeRubyIOReader {
     fn len(&self) -> u64 {
-        self.0.lock().expect("Failed to lock mutex").len()
+        match self.0.lock() {
+            Ok(reader) => reader.len(),
+            Err(_) => {
+                // If the mutex is poisoned, we can't recover, return 0
+                eprintln!("Failed to lock mutex in ThreadSafeRubyIOReader::len");
+                0
+            }
+        }
     }
 }
 
-impl Seek for ThreadSafeRubyReader {
+impl Seek for ThreadSafeRubyIOReader {
     fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
         let mut reader = self
             .0
@@ -262,7 +273,7 @@ impl Seek for ThreadSafeRubyReader {
     }
 }
 
-impl Read for ThreadSafeRubyReader {
+impl Read for ThreadSafeRubyIOReader {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         let mut reader = self
             .0
@@ -272,8 +283,8 @@ impl Read for ThreadSafeRubyReader {
     }
 }
 
-impl ChunkReader for ThreadSafeRubyReader {
-    type T = BufReader<ThreadSafeRubyReader>;
+impl ChunkReader for ThreadSafeRubyIOReader {
+    type T = BufReader<ThreadSafeRubyIOReader>;
 
     fn get_read(&self, start: u64) -> parquet::errors::Result<Self::T> {
         let mut reader = self.clone();
@@ -296,3 +307,126 @@ impl ChunkReader for ThreadSafeRubyReader {
         Ok(buffer.into())
     }
 }
+
+/// Adapter for Ruby IO objects that implements std::io::Write
+pub struct RubyIOWriter {
+    io: Value,
+}
+
+impl RubyIOWriter {
+    pub fn new(io: Value) -> Self {
+        Self { io }
+    }
+}
+
+impl Write for RubyIOWriter {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        // Call Ruby IO#write method
+        let ruby = Ruby::get().map_err(|e| {
+            IoError::new(
+                ErrorKind::Other,
+                format!("Failed to get Ruby runtime: {}", e),
+            )
+        })?;
+
+        if buf.is_empty() {
+            return Ok(0);
+        }
+
+        // Convert bytes to Ruby string
+        let ruby_string = ruby.str_from_slice(buf);
+
+        // Call io.write(string)
+        let result: Result<usize, MagnusError> = self.io.funcall("write", (ruby_string,));
+
+        match result {
+            Ok(bytes_written) => Ok(bytes_written),
+            Err(e) => Err(IoError::new(ErrorKind::Other, e.to_string())),
+        }
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        // Call Ruby IO#flush method
+        let result: Result<Value, MagnusError> = self.io.funcall("flush", ());
+
+        match result {
+            Ok(_) => Ok(()),
+            Err(e) => Err(IoError::new(ErrorKind::Other, e.to_string())),
+        }
+    }
+}
+
+/// Wrapper that implements both Read and Write for Ruby IO objects
+pub struct RubyIO {
+    io: Value,
+}
+
+impl RubyIO {
+    pub fn new(io: Value) -> Self {
+        Self { io }
+    }
+}
+
+impl Read for RubyIO {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        RubyIOReader::new(self.io)?.read(buf)
+    }
+}
+
+impl Write for RubyIO {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        RubyIOWriter::new(self.io).write(buf)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        RubyIOWriter::new(self.io).flush()
+    }
+}
+
+impl Seek for RubyIO {
+    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+        RubyIOReader::new(self.io)?.seek(pos)
+    }
+}
+
+/// Check if a Ruby value responds to IO methods
+pub fn is_io_like(value: Value) -> bool {
+    // Check if the object responds to read/write methods
+    let responds_to_read: Result<bool, MagnusError> = value.funcall("respond_to?", ("read",));
+    let responds_to_write: Result<bool, MagnusError> = value.funcall("respond_to?", ("write",));
+
+    matches!(
+        (responds_to_read, responds_to_write),
+        (Ok(true), _) | (_, Ok(true))
+    )
+}
+
+/// Create a reader from a Ruby IO-like object
+pub fn create_reader(io: Value) -> std::io::Result<RubyIOReader> {
+    // Verify it has a read method
+    let responds_to_read: Result<bool, MagnusError> = io.funcall("respond_to?", ("read",));
+
+    match responds_to_read {
+        Ok(true) => RubyIOReader::new(io),
+        Ok(false) => Err(IoError::new(
+            ErrorKind::InvalidInput,
+            "Object does not respond to 'read' method",
+        )),
+        Err(e) => Err(IoError::new(ErrorKind::Other, e.to_string())),
+    }
+}
+
+/// Create a writer from a Ruby IO-like object
+pub fn create_writer(io: Value) -> std::io::Result<RubyIOWriter> {
+    // Verify it has a write method
+    let responds_to_write: Result<bool, MagnusError> = io.funcall("respond_to?", ("write",));
+
+    match responds_to_write {
+        Ok(true) => Ok(RubyIOWriter::new(io)),
+        Ok(false) => Err(IoError::new(
+            ErrorKind::InvalidInput,
+            "Object does not respond to 'write' method",
+        )),
+        Err(e) => Err(IoError::new(ErrorKind::Other, e.to_string())),
+    }
+}

data/ext/parquet-ruby-adapter/src/lib.rs

@@ -0,0 +1,90 @@
+//! Ruby-specific adapter for parquet-core
+//!
+//! This crate provides Ruby-specific implementations of the parquet-core traits,
+//! enabling seamless integration between Ruby and the core Parquet functionality.
+//!
+//! # Overview
+//!
+//! The adapter implements three main components:
+//!
+//! ## Value Conversion
+//!
+//! The [`RubyValueConverter`] implements the `ValueConverter` trait to handle
+//! conversions between Ruby values (via Magnus) and Parquet values:
+//!
+//! - Ruby integers ↔ Parquet int types
+//! - Ruby floats ↔ Parquet float/double
+//! - Ruby strings ↔ Parquet strings/binary
+//! - Ruby BigDecimal ↔ Parquet decimal types
+//! - Ruby Time/DateTime ↔ Parquet temporal types
+//! - Ruby arrays/hashes ↔ Parquet lists/maps/structs
+//!
+//! ## I/O Operations
+//!
+//! The I/O module provides [`RubyIOReader`] and [`RubyIOWriter`] which implement
+//! parquet-core's `ChunkReader` trait for Ruby IO objects:
+//!
+//! - File objects
+//! - StringIO for in-memory operations
+//! - Any Ruby object implementing read/write/seek methods
+//!
+//! ## Schema Conversion
+//!
+//! Schema utilities for converting between Ruby schema representations and
+//! parquet-core's schema types:
+//!
+//! - Legacy hash-based schemas
+//! - New DSL-based schemas
+//! - Automatic type inference from data
+
+pub mod error;
+pub use error::{ErrorContext, IntoMagnusError, Result, RubyAdapterError};
+
+pub mod chunk_reader;
+pub use chunk_reader::CloneableChunkReader;
+
+pub mod converter;
+pub use converter::RubyValueConverter;
+
+pub mod io;
+pub use io::{create_reader, is_io_like, RubyIO, RubyIOReader, RubyIOWriter};
+
+pub mod logger;
+pub use logger::RubyLogger;
+
+pub mod schema;
+pub use schema::{
+    convert_legacy_schema, extract_field_schemas, is_dsl_schema, parquet_schema_to_ruby,
+    process_schema_value, ruby_schema_to_parquet, RubySchemaBuilder,
+};
+
+pub mod string_cache;
+pub use string_cache::StringCache;
+
+pub mod batch_manager;
+pub use batch_manager::{
+    BatchSizeManager, DEFAULT_MEMORY_THRESHOLD, INITIAL_BATCH_SIZE, MIN_BATCH_SIZE, SAMPLE_SIZE,
+};
+
+pub mod metadata;
+pub use metadata::{parse_metadata, RubyParquetMetaData};
+
+pub mod types;
+pub use types::{
+    ColumnEnumeratorArgs, ParquetWriteArgs, ParserResultType, RowEnumeratorArgs, WriterOutput,
+};
+
+pub mod utils;
+pub use utils::{
+    create_column_enumerator, create_row_enumerator, estimate_parquet_value_size,
+    estimate_row_size, handle_block_or_enum, parse_compression, parse_parquet_write_args,
+};
+
+pub mod reader;
+pub use reader::{each_column, each_row};
+
+pub mod writer;
+pub use writer::{create_writer, finalize_writer, write_columns, write_rows};
+
+pub mod try_into_value;
+pub use try_into_value::TryIntoValue;

data/ext/parquet-ruby-adapter/src/logger.rs

@@ -0,0 +1,64 @@
+use magnus::value::ReprValue;
+use magnus::{Error as MagnusError, Value};
+
+pub struct RubyLogger {
+    logger: Option<Value>,
+}
+
+impl RubyLogger {
+    pub fn new(logger: Option<Value>) -> Result<Self, MagnusError> {
+        // Validate logger has required methods if provided
+        if let Some(ref log) = logger {
+            for method in &["debug", "info", "warn", "error"] {
+                if !log.respond_to(*method, false)? {
+                    return Err(MagnusError::new(
+                        magnus::exception::arg_error(),
+                        format!("Logger must respond to {}", method),
+                    ));
+                }
+            }
+        }
+        Ok(Self { logger })
+    }
+
+    pub fn debug<F: FnOnce() -> String>(&self, msg_fn: F) -> Result<(), MagnusError> {
+        if let Some(ref logger) = self.logger {
+            logger.funcall::<_, _, Value>("debug", (msg_fn(),))?;
+        }
+        Ok(())
+    }
+
+    pub fn info<F: FnOnce() -> String>(&self, msg_fn: F) -> Result<(), MagnusError> {
+        if let Some(ref logger) = self.logger {
+            logger.funcall::<_, _, Value>("info", (msg_fn(),))?;
+        }
+        Ok(())
+    }
+
+    pub fn warn<F: FnOnce() -> String>(&self, msg_fn: F) -> Result<(), MagnusError> {
+        if let Some(ref logger) = self.logger {
+            logger.funcall::<_, _, Value>("warn", (msg_fn(),))?;
+        }
+        Ok(())
+    }
+
+    pub fn error<F: FnOnce() -> String>(&self, msg_fn: F) -> Result<(), MagnusError> {
+        if let Some(ref logger) = self.logger {
+            logger.funcall::<_, _, Value>("error", (msg_fn(),))?;
+        }
+        Ok(())
+    }
+
+    pub fn inner(&self) -> Option<Value> {
+        self.logger
+    }
+}
+
+// Make RubyLogger cloneable for passing to multiple functions
+impl Clone for RubyLogger {
+    fn clone(&self) -> Self {
+        Self {
+            logger: self.logger,
+        }
+    }
+}