parquet 0.5.12 → 0.6.0

data/ext/parquet/src/header_cache.rs

@@ -1,99 +0,0 @@
-/// This module exists to avoid cloning header keys in returned HashMaps.
-/// Since the underlying RString creation already involves cloning,
-/// this caching layer aims to reduce redundant allocations.
-///
-/// Note: Performance testing on macOS showed minimal speed improvements,
-/// so this optimization could be removed if any issues arise.
-use std::{
-    collections::HashMap,
-    sync::{LazyLock, Mutex},
-};
-
-use magnus::{IntoValue, RString, Ruby, Value};
-
-use thiserror::Error;
-
-#[derive(Debug, Clone, Error)]
-pub enum CacheError {
-    #[error("Failed to acquire lock: {0}")]
-    LockError(String),
-    #[error("Failed to convert Ruby String to interned string: {0}")]
-    RStringConversion(String),
-}
-
-static STRING_CACHE: LazyLock<Mutex<HashMap<&'static str, StringCacheKey>>> =
-    LazyLock::new(|| Mutex::new(HashMap::with_capacity(100)));
-
-pub struct StringCache;
-
-#[derive(Copy, Clone)]
-pub struct StringCacheKey(&'static str);
-
-impl StringCacheKey {
-    pub fn new(string: &str) -> Result<Self, CacheError> {
-        let rstr = RString::new(string);
-        let fstr = rstr.to_interned_str();
-        Ok(Self(fstr.as_str().map_err(|e| {
-            CacheError::RStringConversion(e.to_string())
-        })?))
-    }
-}
-
-impl AsRef<str> for StringCacheKey {
-    fn as_ref(&self) -> &'static str {
-        self.0
-    }
-}
-
-impl IntoValue for StringCacheKey {
-    fn into_value_with(self, handle: &Ruby) -> Value {
-        handle.into_value(self.0)
-    }
-}
-
-impl IntoValue for &StringCacheKey {
-    fn into_value_with(self, handle: &Ruby) -> Value {
-        handle.into_value(self.0)
-    }
-}
-
-impl std::fmt::Debug for StringCacheKey {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl PartialEq for StringCacheKey {
-    fn eq(&self, other: &Self) -> bool {
-        self.0 == other.0
-    }
-}
-
-impl std::cmp::Eq for StringCacheKey {}
-
-impl std::hash::Hash for StringCacheKey {
-    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
-        self.0.hash(state);
-    }
-}
-
-impl StringCache {
-    pub fn intern_many<AsStr: AsRef<str>>(
-        strings: &[AsStr],
-    ) -> Result<Vec<StringCacheKey>, CacheError> {
-        let cache = STRING_CACHE
-            .lock()
-            .map_err(|e| CacheError::LockError(e.to_string()))?;
-
-        let mut result: Vec<StringCacheKey> = Vec::with_capacity(strings.len());
-        for string in strings {
-            if let Some((_, interned_string)) = cache.get_key_value(string.as_ref()) {
-                result.push(*interned_string);
-            } else {
-                let interned = StringCacheKey::new(string.as_ref())?;
-                result.push(interned);
-            }
-        }
-        Ok(result)
-    }
-}

data/ext/parquet/src/logger.rs

@@ -1,171 +0,0 @@
-// Logger module for Parquet gem
-// Provides a Rust wrapper for Ruby logger objects
-
-use std::str::FromStr;
-
-use magnus::{exception::runtime_error, value::ReprValue, Error as MagnusError, Ruby, Value};
-
-use crate::{types::ParquetGemError, utils::parse_string_or_symbol};
-
-/// Severity levels that match Ruby's Logger levels
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub enum LogLevel {
-    Debug,
-    Info,
-    Warn,
-    Error,
-    Fatal,
-}
-
-impl FromStr for LogLevel {
-    type Err = MagnusError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Ok(match s {
-            "debug" => LogLevel::Debug,
-            "info" => LogLevel::Info,
-            "warn" => LogLevel::Warn,
-            "error" => LogLevel::Error,
-            "fatal" => LogLevel::Fatal,
-            _ => {
-                return Err(MagnusError::new(
-                    runtime_error(),
-                    format!("Invalid log level: {}", s),
-                ))
-            }
-        })
-    }
-}
-/// A wrapper around a Ruby logger object
-#[derive(Debug, Clone)]
-pub struct RubyLogger {
-    logger: Option<Value>,
-    level: LogLevel,
-}
-
-#[allow(dead_code)]
-impl RubyLogger {
-    pub fn new(ruby: &Ruby, logger_value: Option<Value>) -> Result<Self, ParquetGemError> {
-        let environment_level = std::env::var("PARQUET_GEM_LOG_LEVEL")
-            .unwrap_or_else(|_| "warn".to_string())
-            .parse::<LogLevel>()
-            .unwrap_or(LogLevel::Warn);
-
-        match logger_value {
-            Some(logger) => {
-                if logger.is_nil() {
-                    return Ok(Self {
-                        logger: None,
-                        level: environment_level,
-                    });
-                }
-
-                let level_value = logger.funcall::<_, _, Value>("level", ())?;
-                let level = parse_string_or_symbol(ruby, level_value)?;
-                let level = level
-                    .map(|s| s.parse::<LogLevel>())
-                    .transpose()?
-                    .unwrap_or(environment_level);
-
-                Ok(Self {
-                    logger: Some(logger),
-                    level,
-                })
-            }
-            None => Ok(Self {
-                logger: None,
-                level: environment_level,
-            }),
-        }
-    }
-
-    /// Log a message at the given level
-    pub fn log(&self, level: LogLevel, message: &str) -> Result<(), MagnusError> {
-        let method = match level {
-            LogLevel::Debug => "debug",
-            LogLevel::Info => "info",
-            LogLevel::Warn => "warn",
-            LogLevel::Error => "error",
-            LogLevel::Fatal => "fatal",
-        };
-
-        match self.logger {
-            Some(logger) => {
-                logger.funcall::<_, _, Value>(method, (message,))?;
-            }
-            None => eprintln!("{}", message),
-        }
-
-        Ok(())
-    }
-
-    /// Log a debug message
-    pub fn debug<F, S>(&self, message_fn: F) -> Result<(), MagnusError>
-    where
-        F: FnOnce() -> S,
-        S: AsRef<str>,
-    {
-        if self.level <= LogLevel::Debug {
-            let message = message_fn();
-            self.log(LogLevel::Debug, message.as_ref())
-        } else {
-            Ok(())
-        }
-    }
-
-    /// Log an info message
-    pub fn info<F, S>(&self, message_fn: F) -> Result<(), MagnusError>
-    where
-        F: FnOnce() -> S,
-        S: AsRef<str>,
-    {
-        if self.level <= LogLevel::Info {
-            let message = message_fn();
-            self.log(LogLevel::Info, message.as_ref())
-        } else {
-            Ok(())
-        }
-    }
-
-    /// Log a warning message
-    pub fn warn<F, S>(&self, message_fn: F) -> Result<(), MagnusError>
-    where
-        F: FnOnce() -> S,
-        S: AsRef<str>,
-    {
-        if self.level <= LogLevel::Warn {
-            let message = message_fn();
-            self.log(LogLevel::Warn, message.as_ref())
-        } else {
-            Ok(())
-        }
-    }
-
-    /// Log an error message
-    pub fn error<F, S>(&self, message_fn: F) -> Result<(), MagnusError>
-    where
-        F: FnOnce() -> S,
-        S: AsRef<str>,
-    {
-        if self.level <= LogLevel::Error {
-            let message = message_fn();
-            self.log(LogLevel::Error, message.as_ref())
-        } else {
-            Ok(())
-        }
-    }
-
-    /// Log a fatal message
-    pub fn fatal<F, S>(&self, message_fn: F) -> Result<(), MagnusError>
-    where
-        F: FnOnce() -> S,
-        S: AsRef<str>,
-    {
-        if self.level <= LogLevel::Fatal {
-            let message = message_fn();
-            self.log(LogLevel::Fatal, message.as_ref())
-        } else {
-            Ok(())
-        }
-    }
-}

data/ext/parquet/src/reader/common.rs

@@ -1,111 +0,0 @@
-use ahash::RandomState;
-use arrow_schema::Schema;
-use either::Either;
-use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
-use parquet::arrow::ProjectionMask;
-use std::collections::HashMap;
-use std::fs::File;
-use std::rc::Rc;
-use std::sync::Arc;
-
-use magnus::value::ReprValue;
-use magnus::{Error as MagnusError, Ruby, Value};
-
-use crate::header_cache::StringCache;
-use crate::ruby_reader::{RubyReader, ThreadSafeRubyReader};
-use crate::types::{ParquetGemError, TryIntoValue};
-use crate::ColumnRecord;
-
-/// Opens a parquet file or IO-like object for reading
-///
-/// This function handles both file paths (as strings) and IO-like objects,
-/// returning either a File or a ThreadSafeRubyReader that can be used with
-/// parquet readers.
-pub fn open_parquet_source(
-    ruby: Rc<Ruby>,
-    to_read: Value,
-) -> Result<Either<File, ThreadSafeRubyReader>, ParquetGemError> {
-    if to_read.is_kind_of(ruby.class_string()) {
-        let path_string = to_read.to_r_string()?;
-        let file_path = unsafe { path_string.as_str()? };
-        let file = File::open(file_path).map_err(ParquetGemError::from)?;
-        Ok(Either::Left(file))
-    } else {
-        let readable = ThreadSafeRubyReader::new(RubyReader::new(ruby, to_read)?);
-        Ok(Either::Right(readable))
-    }
-}
-
-/// Helper function to check if a block is given and create an appropriate enumerator
-/// if not
-pub fn handle_block_or_enum<F, T>(
-    _ruby: &magnus::Ruby,
-    block_given: bool,
-    create_enum: F,
-) -> Result<Option<T>, MagnusError>
-where
-    F: FnOnce() -> Result<T, MagnusError>,
-{
-    if !block_given {
-        let enum_value = create_enum()?;
-        return Ok(Some(enum_value));
-    }
-    Ok(None)
-}
-
-/// Creates a ParquetRecordBatchReader with the given columns and batch size configurations
-pub fn create_batch_reader<T: parquet::file::reader::ChunkReader + 'static>(
-    reader: T,
-    columns: &Option<Vec<String>>,
-    batch_size: Option<usize>,
-) -> Result<(ParquetRecordBatchReader, std::sync::Arc<Schema>, i64), ParquetGemError> {
-    let mut builder =
-        ParquetRecordBatchReaderBuilder::try_new(reader).map_err(ParquetGemError::Parquet)?;
-
-    let schema = builder.schema().clone();
-    let num_rows = builder.metadata().file_metadata().num_rows();
-
-    // If columns are specified, project only those columns
-    if let Some(cols) = columns {
-        // Get the parquet schema
-        let parquet_schema = builder.parquet_schema();
-
-        // Create a projection mask from column names
-        let projection = ProjectionMask::columns(parquet_schema, cols.iter().map(|s| s.as_str()));
-        builder = builder.with_projection(projection);
-    }
-
-    if let Some(batch_size) = batch_size {
-        builder = builder.with_batch_size(batch_size);
-    }
-
-    let reader = builder.build().map_err(ParquetGemError::Parquet)?;
-    Ok((reader, schema, num_rows))
-}
-
-/// Handles the case of an empty parquet file (no rows) by yielding a record with empty arrays
-/// Returns true if the file was empty and was handled, false otherwise
-pub fn handle_empty_file(
-    ruby: &magnus::Ruby,
-    schema: &Arc<Schema>,
-    num_rows: i64,
-) -> Result<bool, ParquetGemError> {
-    if num_rows == 0 {
-        let mut map =
-            HashMap::with_capacity_and_hasher(schema.fields().len(), RandomState::default());
-        let headers: Vec<String> = schema
-            .fields()
-            .iter()
-            .map(|field| field.name().to_string())
-            .collect();
-        let interned_headers =
-            StringCache::intern_many(&headers).map_err(ParquetGemError::HeaderIntern)?;
-        for field in interned_headers.iter() {
-            map.insert(*field, vec![]);
-        }
-        let record = ColumnRecord::Map(map);
-        let _: Value = ruby.yield_value(record.try_into_value_with(ruby)?)?;
-        return Ok(true);
-    }
-    Ok(false)
-}

data/ext/parquet/src/reader/mod.rs

@@ -1,211 +0,0 @@
-mod common;
-mod parquet_column_reader;
-mod parquet_row_reader;
-mod unified;
-use std::{fs::File, rc::Rc};
-
-use magnus::{value::ReprValue, Error as MagnusError, Ruby, Value};
-use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
-pub use parquet_column_reader::parse_parquet_columns;
-pub use parquet_row_reader::parse_parquet_rows;
-
-use crate::{
-    ruby_reader::{RubyReader, ThreadSafeRubyReader},
-    types::{ParquetGemError, TryIntoValue},
-};
-
-struct RubyParquetMetaData(ParquetMetaData);
-
-impl TryIntoValue for RubyParquetMetaData {
-    fn try_into_value_with(self, handle: &Ruby) -> Result<Value, ParquetGemError> {
-        let metadata = self.0;
-        let file_metadata = metadata.file_metadata();
-        let row_groups = metadata.row_groups();
-
-        // Construct a hash with the metadata
-        let hash = handle.hash_new();
-        hash.aset("num_rows", file_metadata.num_rows())?;
-        hash.aset("created_by", file_metadata.created_by())?;
-        // Convert key_value_metadata to a Ruby array if it exists
-        if let Some(key_value_metadata) = file_metadata.key_value_metadata() {
-            let kv_array = handle.ary_new();
-            for kv in key_value_metadata {
-                let kv_hash = handle.hash_new();
-                kv_hash.aset("key", kv.key.clone())?;
-                kv_hash.aset("value", kv.value.clone())?;
-                kv_array.push(kv_hash)?;
-            }
-            hash.aset("key_value_metadata", kv_array)?;
-        } else {
-            hash.aset("key_value_metadata", None::<Value>)?;
-        }
-
-        // Convert schema to a Ruby hash since &Type doesn't implement IntoValue
-        let schema_hash = handle.hash_new();
-        let schema = file_metadata.schema();
-        schema_hash.aset("name", schema.name())?;
-        // Add schema fields information
-        let fields_array = handle.ary_new();
-        for field in schema.get_fields() {
-            let field_hash = handle.hash_new();
-            field_hash.aset("name", field.name())?;
-
-            // Handle different field types
-            match field.as_ref() {
-                parquet::schema::types::Type::PrimitiveType {
-                    physical_type,
-                    type_length,
-                    scale,
-                    precision,
-                    ..
-                } => {
-                    field_hash.aset("type", "primitive")?;
-                    field_hash.aset("physical_type", format!("{:?}", physical_type))?;
-                    field_hash.aset("type_length", *type_length)?;
-                    field_hash.aset("scale", *scale)?;
-                    field_hash.aset("precision", *precision)?;
-                }
-                parquet::schema::types::Type::GroupType { .. } => {
-                    field_hash.aset("type", "group")?;
-                }
-            }
-
-            // Add basic info
-            let basic_info = field.get_basic_info();
-            field_hash.aset("repetition", format!("{:?}", basic_info.repetition()))?;
-            field_hash.aset(
-                "converted_type",
-                format!("{:?}", basic_info.converted_type()),
-            )?;
-            if let Some(logical_type) = basic_info.logical_type() {
-                field_hash.aset("logical_type", format!("{:?}", logical_type))?;
-            }
-
-            fields_array.push(field_hash)?;
-        }
-        schema_hash.aset("fields", fields_array)?;
-
-        hash.aset("schema", schema_hash)?;
-
-        // Convert row_groups to a Ruby array since &[RowGroupMetaData] doesn't implement IntoValue
-        let row_groups_array = handle.ary_new();
-        for row_group in row_groups.iter() {
-            let rg_hash = handle.hash_new();
-            rg_hash.aset("num_columns", row_group.num_columns())?;
-            rg_hash.aset("num_rows", row_group.num_rows())?;
-            rg_hash.aset("total_byte_size", row_group.total_byte_size())?;
-            rg_hash.aset("file_offset", row_group.file_offset())?;
-            rg_hash.aset("ordinal", row_group.ordinal())?;
-            rg_hash.aset("compressed_size", row_group.compressed_size())?;
-
-            // Add column chunks metadata
-            let columns_array = handle.ary_new();
-            for col_idx in 0..row_group.num_columns() {
-                let column = row_group.column(col_idx);
-                let col_hash = handle.hash_new();
-
-                col_hash.aset("column_path", column.column_path().string())?;
-                col_hash.aset("file_path", column.file_path())?;
-                col_hash.aset("file_offset", column.file_offset())?;
-                col_hash.aset("num_values", column.num_values())?;
-                col_hash.aset("compression", format!("{:?}", column.compression()))?;
-                col_hash.aset("total_compressed_size", column.compressed_size())?;
-                col_hash.aset("total_uncompressed_size", column.uncompressed_size())?;
-                col_hash.aset("data_page_offset", column.data_page_offset())?;
-
-                if let Some(offset) = column.dictionary_page_offset() {
-                    col_hash.aset("dictionary_page_offset", offset)?;
-                }
-
-                if let Some(offset) = column.bloom_filter_offset() {
-                    col_hash.aset("bloom_filter_offset", offset)?;
-                }
-
-                if let Some(length) = column.bloom_filter_length() {
-                    col_hash.aset("bloom_filter_length", length)?;
-                }
-
-                if let Some(offset) = column.offset_index_offset() {
-                    col_hash.aset("offset_index_offset", offset)?;
-                }
-
-                if let Some(length) = column.offset_index_length() {
-                    col_hash.aset("offset_index_length", length)?;
-                }
-
-                if let Some(offset) = column.column_index_offset() {
-                    col_hash.aset("column_index_offset", offset)?;
-                }
-
-                if let Some(length) = column.column_index_length() {
-                    col_hash.aset("column_index_length", length)?;
-                }
-
-                // Add encodings
-                let encodings_array = handle.ary_new();
-                for encoding in column.encodings() {
-                    encodings_array.push(format!("{:?}", encoding))?;
-                }
-                col_hash.aset("encodings", encodings_array)?;
-
-                // Add statistics if available
-                if let Some(stats) = column.statistics() {
-                    let stats_hash = handle.hash_new();
-                    stats_hash.aset("min_is_exact", stats.min_is_exact())?;
-                    stats_hash.aset("max_is_exact", stats.max_is_exact())?;
-
-                    col_hash.aset("statistics", stats_hash)?;
-                }
-
-                // Add page encoding stats if available
-                if let Some(page_encoding_stats) = column.page_encoding_stats() {
-                    let page_stats_array = handle.ary_new();
-                    for stat in page_encoding_stats {
-                        let stat_hash = handle.hash_new();
-                        stat_hash.aset("page_type", format!("{:?}", stat.page_type))?;
-                        stat_hash.aset("encoding", format!("{:?}", stat.encoding))?;
-                        stat_hash.aset("count", stat.count)?;
-                        page_stats_array.push(stat_hash)?;
-                    }
-                    col_hash.aset("page_encoding_stats", page_stats_array)?;
-                }
-
-                columns_array.push(col_hash)?;
-            }
-            rg_hash.aset("columns", columns_array)?;
-
-            row_groups_array.push(rg_hash)?;
-        }
-        hash.aset("row_groups", row_groups_array)?;
-
-        Ok(handle.into_value(hash))
-    }
-}
-
-pub fn parse_metadata(_rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
-    let ruby = unsafe { Ruby::get_unchecked() };
-
-    if args.len() != 1 {
-        return Err(MagnusError::new(
-            magnus::exception::arg_error(),
-            format!("metadata expects exactly 1 argument (file path or IO-like object), got {}", args.len()),
-        ));
-    }
-
-    let ruby = Rc::new(ruby);
-    let arg = args[0];
-
-    let mut reader = ParquetMetaDataReader::new();
-    if arg.is_kind_of(ruby.class_string()) {
-        let path = arg.to_r_string()?.to_string()?;
-        let file = File::open(path).map_err(ParquetGemError::FileOpen)?;
-        reader.try_parse(&file).map_err(ParquetGemError::Parquet)?;
-    } else {
-        let file = ThreadSafeRubyReader::new(RubyReader::new(ruby.clone(), arg)?);
-        reader.try_parse(&file).map_err(ParquetGemError::Parquet)?;
-    }
-
-    let metadata = reader.finish().map_err(ParquetGemError::Parquet)?;
-
-    Ok(RubyParquetMetaData(metadata).try_into_value_with(&ruby)?)
-}

data/ext/parquet/src/reader/parquet_column_reader.rs

@@ -1,44 +0,0 @@
-use crate::reader::unified::{parse_parquet_unified, ParserType, UnifiedParserArgs};
-use crate::utils::*;
-use crate::ParquetGemError;
-
-use magnus::{Error as MagnusError, Ruby, Value};
-use std::rc::Rc;
-
-#[inline]
-pub fn parse_parquet_columns(rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
-    let ruby = unsafe { Ruby::get_unchecked() };
-    parse_parquet_columns_impl(Rc::new(ruby), rb_self, args).map_err(|e| {
-        let z: MagnusError = e.into();
-        z
-    })
-}
-
-#[inline]
-fn parse_parquet_columns_impl(
-    ruby: Rc<Ruby>,
-    rb_self: Value,
-    args: &[Value],
-) -> Result<Value, ParquetGemError> {
-    let ParquetColumnsArgs {
-        to_read,
-        result_type,
-        columns,
-        batch_size,
-        strict,
-        logger,
-    } = parse_parquet_columns_args(&ruby, args)?;
-
-    // Use the unified parsing implementation
-    parse_parquet_unified(
-        ruby,
-        rb_self,
-        UnifiedParserArgs {
-            to_read,
-            result_type,
-            columns,
-            parser_type: ParserType::Column { batch_size, strict },
-            logger,
-        },
-    )
-}

data/ext/parquet/src/reader/parquet_row_reader.rs

@@ -1,43 +0,0 @@
-use crate::reader::unified::{parse_parquet_unified, ParserType, UnifiedParserArgs};
-use crate::utils::*;
-use crate::ParquetGemError;
-
-use magnus::{Error as MagnusError, Ruby, Value};
-use std::rc::Rc;
-
-#[inline]
-pub fn parse_parquet_rows(rb_self: Value, args: &[Value]) -> Result<Value, MagnusError> {
-    let ruby = unsafe { Ruby::get_unchecked() };
-    parse_parquet_rows_impl(Rc::new(ruby), rb_self, args).map_err(|e| {
-        let z: MagnusError = e.into();
-        z
-    })
-}
-
-#[inline]
-fn parse_parquet_rows_impl(
-    ruby: Rc<Ruby>,
-    rb_self: Value,
-    args: &[Value],
-) -> Result<Value, ParquetGemError> {
-    let ParquetRowsArgs {
-        to_read,
-        result_type,
-        columns,
-        strict,
-        logger,
-    } = parse_parquet_rows_args(&ruby, args)?;
-
-    // Use the unified parsing implementation
-    parse_parquet_unified(
-        ruby,
-        rb_self,
-        UnifiedParserArgs {
-            to_read,
-            result_type,
-            columns,
-            parser_type: ParserType::Row { strict },
-            logger,
-        },
-    )
-}