parquet 0.5.10 → 0.5.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 114891cfa5fa190e1f00d44803327f1c90cc11f64ba23f7f2a9cc9f9379da787
-  data.tar.gz: 9168b2be960faa93ce9c84d170c6e8f73819535bcedbf3d3b26869ff9829ecc6
+  metadata.gz: 82528b663c4a577262db90b6d17ba473a81d0ea725ceba486b63a3619040fa73
+  data.tar.gz: 2e44daa9b4e36ef1503589daaa0815cbc3acee10c565d9942f6c0b6d35ced5f0
 SHA512:
-  metadata.gz: f07f99a188ac5fa0663616fba00b1990a2cbd6bb14462383915f0e1617c26c5ca481840c16179958f2b3760b334f176e2e4542d95e3cc922379948ac2b0bfa61
-  data.tar.gz: 42c7b0779d6e3fa46addc5fa92420f326418a54962d391e9b063db8378f8a5f8c2916b43f356649fc127e8fc582aa1e98d7afd71f0bc5f9700a0664ed46313f6
+  metadata.gz: 418951253384f5492385fcb30fa5b0113b85d9bc51346b6abad16105c124d8869266943c1a29bc0879cfee4270b94d32fb99004e233c6ebde4a70e1d329435af
+  data.tar.gz: bc0db4ebb36add314253b5b9b946cc2c84f315d51ba7fefbead6c7de3b65a3f7752fa4e4cf0be19704405b390ae0106d8383e30791e7fac4a86a75141c214de1

data/ext/parquet/src/reader/common.rs

@@ -12,81 +12,27 @@ use magnus::value::ReprValue;
 use magnus::{Error as MagnusError, Ruby, Value};
 
 use crate::header_cache::StringCache;
-use crate::logger::RubyLogger;
 use crate::ruby_reader::{RubyReader, ThreadSafeRubyReader};
 use crate::types::{ParquetGemError, TryIntoValue};
 use crate::ColumnRecord;
 
-use super::format_detector::{detect_file_format, detect_format_from_extension, FileFormat};
-
-/// Represents the different data sources we can open
-pub enum DataSource {
-    Parquet(Either<File, ThreadSafeRubyReader>),
-    Arrow(Either<File, ThreadSafeRubyReader>),
-}
-
-/// Opens a data file (Parquet or Arrow) for reading, automatically detecting the format
-pub fn open_data_source(
+/// Opens a parquet file or IO-like object for reading
+///
+/// This function handles both file paths (as strings) and IO-like objects,
+/// returning either a File or a ThreadSafeRubyReader that can be used with
+/// parquet readers.
+pub fn open_parquet_source(
     ruby: Rc<Ruby>,
     to_read: Value,
-    ruby_logger: &RubyLogger,
-) -> Result<DataSource, ParquetGemError> {
+) -> Result<Either<File, ThreadSafeRubyReader>, ParquetGemError> {
     if to_read.is_kind_of(ruby.class_string()) {
         let path_string = to_read.to_r_string()?;
         let file_path = unsafe { path_string.as_str()? };
-
-        // Try to detect format from extension first
-        let format_hint = detect_format_from_extension(file_path);
-
-        let mut file = File::open(file_path).map_err(ParquetGemError::from)?;
-
-        // Detect actual format from file content
-        let format = detect_file_format(&mut file)?;
-
-        // Warn if extension doesn't match content
-        if let Some(hint) = format_hint {
-            if hint != format {
-                ruby_logger.warn(|| {
-                    format!(
-                        "Extension implied format {:?} but actual format is {:?}",
-                        hint, format
-                    )
-                })?;
-            }
-        }
-
-        match format {
-            FileFormat::Parquet => Ok(DataSource::Parquet(Either::Left(file))),
-            FileFormat::Arrow => Ok(DataSource::Arrow(Either::Left(file))),
-        }
+        let file = File::open(file_path).map_err(ParquetGemError::from)?;
+        Ok(Either::Left(file))
     } else {
-        // For IO-like objects, we need to use a temporary file
-        use std::io::{Read, Write};
-        use tempfile::NamedTempFile;
-
-        let mut readable = RubyReader::new(ruby.clone(), to_read)?;
-        let mut temp_file = NamedTempFile::new().map_err(ParquetGemError::from)?;
-
-        // Copy the entire content to the temporary file
-        let mut buffer = vec![0u8; 8192];
-        loop {
-            let bytes_read = readable.read(&mut buffer)?;
-            if bytes_read == 0 {
-                break;
-            }
-            temp_file.write_all(&buffer[..bytes_read])?;
-        }
-        temp_file.flush()?;
-
-        // Detect format from the temporary file
-        let mut file = temp_file.reopen()?;
-        let format = detect_file_format(&mut file)?;
-
-        // Use the temporary file as the source
-        match format {
-            FileFormat::Parquet => Ok(DataSource::Parquet(Either::Left(file))),
-            FileFormat::Arrow => Ok(DataSource::Arrow(Either::Left(file))),
-        }
+        let readable = ThreadSafeRubyReader::new(RubyReader::new(ruby, to_read)?);
+        Ok(Either::Right(readable))
     }
 }
 

data/ext/parquet/src/reader/mod.rs

@@ -1,6 +1,4 @@
-mod arrow_reader;
 mod common;
-mod format_detector;
 mod parquet_column_reader;
 mod parquet_row_reader;
 mod unified;
@@ -190,10 +188,7 @@ pub fn parse_metadata(_rb_self: Value, args: &[Value]) -> Result<Value, MagnusEr
     if args.len() != 1 {
         return Err(MagnusError::new(
             magnus::exception::arg_error(),
-            format!(
-                "metadata expects exactly 1 argument (file path or IO-like object), got {}",
-                args.len()
-            ),
+            format!("metadata expects exactly 1 argument (file path or IO-like object), got {}", args.len()),
         ));
     }
 
@@ -213,4 +208,4 @@ pub fn parse_metadata(_rb_self: Value, args: &[Value]) -> Result<Value, MagnusEr
     let metadata = reader.finish().map_err(ParquetGemError::Parquet)?;
 
     Ok(RubyParquetMetaData(metadata).try_into_value_with(&ruby)?)
-}
+}

data/ext/parquet/src/reader/unified/mod.rs

@@ -13,11 +13,8 @@ use std::collections::HashMap;
 use std::rc::Rc;
 use std::sync::OnceLock;
 
-use super::arrow_reader::{
-    process_arrow_column_data, process_arrow_file_column_data, process_arrow_row_data,
-};
 use super::common::{
-    create_batch_reader, handle_block_or_enum, handle_empty_file, open_data_source, DataSource,
+    create_batch_reader, handle_block_or_enum, handle_empty_file, open_parquet_source,
 };
 use crate::types::ArrayWrapper;
 
@@ -103,99 +100,34 @@ pub fn parse_parquet_unified(
         }
     }
 
-    // Open the data source and detect format
-    let source = open_data_source(ruby.clone(), to_read, &ruby_logger)?;
+    // Open the Parquet source
+    let source = open_parquet_source(ruby.clone(), to_read)?;
 
-    // Based on the source format and parser type, handle the data differently
-    match (source, &parser_type) {
-        (DataSource::Parquet(reader), ParserType::Row { strict }) => {
-            // Handle Parquet row-based parsing
+    // Based on the parser type, handle the data differently
+    match parser_type {
+        ParserType::Row { strict } => {
+            // Handle row-based parsing
             process_row_data(
                 ruby.clone(),
-                reader,
+                source,
                 &columns,
                 result_type,
-                *strict,
+                strict,
                 &ruby_logger,
             )?;
         }
-        (DataSource::Parquet(reader), ParserType::Column { batch_size, strict }) => {
-            // Handle Parquet column-based parsing
+        ParserType::Column { batch_size, strict } => {
+            // Handle column-based parsing
             process_column_data(
                 ruby.clone(),
-                reader,
+                source,
                 &columns,
                 result_type,
-                *batch_size,
-                *strict,
+                batch_size,
+                strict,
                 &ruby_logger,
             )?;
         }
-        (DataSource::Arrow(reader), ParserType::Row { strict }) => {
-            // Handle Arrow row-based parsing
-            match reader {
-                Either::Left(file) => {
-                    // For seekable files, use FileReader which handles IPC file format
-                    use arrow_ipc::reader::FileReader;
-                    let file_reader = FileReader::try_new(file, None)
-                        .map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                    use super::arrow_reader::process_arrow_file_row_data;
-                    process_arrow_file_row_data(
-                        ruby.clone(),
-                        file_reader,
-                        &columns,
-                        result_type,
-                        *strict,
-                        &ruby_logger,
-                    )?;
-                }
-                Either::Right(readable) => {
-                    use arrow_ipc::reader::StreamReader;
-                    let stream_reader = StreamReader::try_new(readable, None)
-                        .map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-                    process_arrow_row_data(
-                        ruby.clone(),
-                        stream_reader,
-                        &columns,
-                        result_type,
-                        *strict,
-                        &ruby_logger,
-                    )?;
-                }
-            }
-        }
-        (DataSource::Arrow(reader), ParserType::Column { batch_size, strict }) => {
-            // Handle Arrow column-based parsing
-            match reader {
-                Either::Left(file) => {
-                    // For seekable files, we can use the optimized FileReader
-                    process_arrow_file_column_data(
-                        ruby.clone(),
-                        file,
-                        &columns,
-                        result_type,
-                        *batch_size,
-                        *strict,
-                        &ruby_logger,
-                    )?;
-                }
-                Either::Right(readable) => {
-                    use arrow_ipc::reader::StreamReader;
-                    let stream_reader = StreamReader::try_new(readable, None)
-                        .map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-                    process_arrow_column_data(
-                        ruby.clone(),
-                        stream_reader,
-                        &columns,
-                        result_type,
-                        *batch_size,
-                        *strict,
-                        &ruby_logger,
-                    )?;
-                }
-            }
-        }
     }
 
     Ok(ruby.qnil().into_value_with(&ruby))

data/ext/parquet/src/types/mod.rs

@@ -55,10 +55,6 @@ pub enum ParquetGemError {
     Parquet(#[from] parquet::errors::ParquetError),
     #[error("Arrow error: {0}")]
     Arrow(#[from] arrow_schema::ArrowError),
-    #[error("Arrow IPC error: {0}")]
-    ArrowIpc(String),
-    #[error("Unknown file format")]
-    UnknownFormat,
     #[error("UTF-8 error: {0}")]
     Utf8Error(#[from] simdutf8::basic::Utf8Error),
     #[error("Jiff error: {0}")]

data/lib/parquet/version.rb

@@ -1,3 +1,3 @@
 module Parquet
-  VERSION = "0.5.10"
+  VERSION = "0.5.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parquet
 version: !ruby/object:Gem::Version
-  version: 0.5.10
+  version: 0.5.11
 platform: ruby
 authors:
 - Nathan Jaremko
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-17 00:00:00.000000000 Z
+date: 2025-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -62,9 +62,7 @@ files:
 - ext/parquet/src/header_cache.rs
 - ext/parquet/src/lib.rs
 - ext/parquet/src/logger.rs
-- ext/parquet/src/reader/arrow_reader.rs
 - ext/parquet/src/reader/common.rs
-- ext/parquet/src/reader/format_detector.rs
 - ext/parquet/src/reader/mod.rs
 - ext/parquet/src/reader/parquet_column_reader.rs
 - ext/parquet/src/reader/parquet_row_reader.rs

data/ext/parquet/src/reader/arrow_reader.rs

@@ -1,579 +0,0 @@
-use crate::header_cache::StringCache;
-use crate::logger::RubyLogger;
-use crate::types::ArrayWrapper;
-use crate::types::{
-    ColumnRecord, ParquetGemError, ParquetValueVec, ParserResultType, RowRecord, TryIntoValue,
-};
-use ahash::RandomState;
-use arrow_array::RecordBatch;
-use arrow_ipc::reader::{FileReader, StreamReader};
-use arrow_schema::Schema;
-use magnus::{Ruby, Value};
-use std::collections::HashMap;
-use std::fs::File;
-use std::io::Read;
-use std::rc::Rc;
-use std::sync::{Arc, OnceLock};
-
-/// Process Arrow IPC file data for column-based parsing
-pub fn process_arrow_column_data<R: Read>(
-    ruby: Rc<Ruby>,
-    reader: StreamReader<R>,
-    columns: &Option<Vec<String>>,
-    result_type: ParserResultType,
-    _batch_size: Option<usize>,
-    strict: bool,
-    ruby_logger: &RubyLogger,
-) -> Result<(), ParquetGemError> {
-    let schema = reader.schema();
-    ruby_logger.debug(|| format!("Arrow schema loaded: {:?}", schema))?;
-
-    // Filter schema if columns are specified
-    let _filtered_schema = if let Some(cols) = columns {
-        let mut fields = Vec::new();
-        for field in schema.fields() {
-            if cols.contains(&field.name().to_string()) {
-                fields.push(field.clone());
-            }
-        }
-        Arc::new(Schema::new(fields))
-    } else {
-        schema.clone()
-    };
-
-    match result_type {
-        ParserResultType::Hash => {
-            let headers = OnceLock::new();
-
-            for batch_result in reader {
-                let batch = batch_result.map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                // Filter columns if needed
-                let batch = if let Some(cols) = columns {
-                    filter_record_batch(&batch, cols)?
-                } else {
-                    batch
-                };
-
-                let local_headers = headers
-                    .get_or_init(|| {
-                        let schema = batch.schema();
-                        let fields = schema.fields();
-                        let mut header_string = Vec::with_capacity(fields.len());
-                        for field in fields {
-                            header_string.push(field.name().to_owned());
-                        }
-                        StringCache::intern_many(&header_string)
-                    })
-                    .as_ref()
-                    .map_err(|e| ParquetGemError::HeaderIntern(e.clone()))?;
-
-                let mut map =
-                    HashMap::with_capacity_and_hasher(local_headers.len(), RandomState::default());
-
-                batch
-                    .columns()
-                    .iter()
-                    .enumerate()
-                    .try_for_each(|(i, column)| {
-                        let header = local_headers[i];
-                        let values = ParquetValueVec::try_from(ArrayWrapper {
-                            array: column,
-                            strict,
-                        })?;
-                        map.insert(header, values.into_inner());
-                        Ok::<_, ParquetGemError>(())
-                    })?;
-
-                let record = ColumnRecord::Map::<RandomState>(map);
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-        ParserResultType::Array => {
-            for batch_result in reader {
-                let batch = batch_result.map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                // Filter columns if needed
-                let batch = if let Some(cols) = columns {
-                    filter_record_batch(&batch, cols)?
-                } else {
-                    batch
-                };
-
-                let vec = batch
-                    .columns()
-                    .iter()
-                    .map(|column| {
-                        let values = ParquetValueVec::try_from(ArrayWrapper {
-                            array: column,
-                            strict,
-                        })?;
-                        Ok::<_, ParquetGemError>(values.into_inner())
-                    })
-                    .collect::<Result<Vec<_>, _>>()?;
-
-                let record = ColumnRecord::Vec::<RandomState>(vec);
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-    }
-
-    Ok(())
-}
-
-/// Process Arrow IPC file data for row-based parsing
-pub fn process_arrow_row_data<R: Read>(
-    ruby: Rc<Ruby>,
-    reader: StreamReader<R>,
-    columns: &Option<Vec<String>>,
-    result_type: ParserResultType,
-    strict: bool,
-    ruby_logger: &RubyLogger,
-) -> Result<(), ParquetGemError> {
-    let schema = reader.schema();
-    ruby_logger.debug(|| format!("Arrow schema loaded: {:?}", schema))?;
-
-    match result_type {
-        ParserResultType::Hash => {
-            let headers = OnceLock::new();
-
-            for batch_result in reader {
-                let batch = batch_result.map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                // Filter columns if needed
-                let batch = if let Some(cols) = columns {
-                    filter_record_batch(&batch, cols)?
-                } else {
-                    batch
-                };
-
-                let local_headers = headers
-                    .get_or_init(|| {
-                        let schema = batch.schema();
-                        let fields = schema.fields();
-                        let mut header_string = Vec::with_capacity(fields.len());
-                        for field in fields {
-                            header_string.push(field.name().to_owned());
-                        }
-                        StringCache::intern_many(&header_string)
-                    })
-                    .as_ref()
-                    .map_err(|e| ParquetGemError::HeaderIntern(e.clone()))?;
-
-                // Convert columnar data to rows
-                for row_idx in 0..batch.num_rows() {
-                    let mut map = HashMap::with_capacity_and_hasher(
-                        local_headers.len(),
-                        RandomState::default(),
-                    );
-
-                    for (col_idx, column) in batch.columns().iter().enumerate() {
-                        let header = local_headers[col_idx];
-                        let value = extract_value_at_index(column, row_idx, strict)?;
-                        map.insert(header, value);
-                    }
-
-                    let record = RowRecord::Map::<RandomState>(map);
-                    let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-                }
-            }
-        }
-        ParserResultType::Array => {
-            for batch_result in reader {
-                let batch = batch_result.map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                // Filter columns if needed
-                let batch = if let Some(cols) = columns {
-                    filter_record_batch(&batch, cols)?
-                } else {
-                    batch
-                };
-
-                // Convert columnar data to rows
-                for row_idx in 0..batch.num_rows() {
-                    let mut row_vec = Vec::with_capacity(batch.num_columns());
-
-                    for column in batch.columns() {
-                        let value = extract_value_at_index(column, row_idx, strict)?;
-                        row_vec.push(value);
-                    }
-
-                    let record = RowRecord::Vec::<RandomState>(row_vec);
-                    let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-                }
-            }
-        }
-    }
-
-    Ok(())
-}
-
-/// Process Arrow IPC file with FileReader for row-based parsing
-pub fn process_arrow_file_row_data(
-    ruby: Rc<Ruby>,
-    reader: FileReader<File>,
-    columns: &Option<Vec<String>>,
-    result_type: ParserResultType,
-    strict: bool,
-    ruby_logger: &RubyLogger,
-) -> Result<(), ParquetGemError> {
-    let schema = reader.schema();
-    ruby_logger.debug(|| format!("Arrow file schema loaded: {:?}", schema))?;
-
-    match result_type {
-        ParserResultType::Hash => {
-            let headers = OnceLock::new();
-
-            for batch_result in reader {
-                let batch = batch_result.map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                // Filter columns if needed
-                let batch = if let Some(cols) = columns {
-                    filter_record_batch(&batch, cols)?
-                } else {
-                    batch
-                };
-
-                let local_headers = headers
-                    .get_or_init(|| {
-                        let schema = batch.schema();
-                        let fields = schema.fields();
-                        let mut header_string = Vec::with_capacity(fields.len());
-                        for field in fields {
-                            header_string.push(field.name().to_owned());
-                        }
-                        StringCache::intern_many(&header_string)
-                    })
-                    .as_ref()
-                    .map_err(|e| ParquetGemError::HeaderIntern(e.clone()))?;
-
-                // Convert columnar data to rows
-                for row_idx in 0..batch.num_rows() {
-                    let mut map = HashMap::with_capacity_and_hasher(
-                        local_headers.len(),
-                        RandomState::default(),
-                    );
-
-                    for (col_idx, column) in batch.columns().iter().enumerate() {
-                        let header = local_headers[col_idx];
-                        let value = extract_value_at_index(column, row_idx, strict)?;
-                        map.insert(header, value);
-                    }
-
-                    let record = RowRecord::Map::<RandomState>(map);
-                    let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-                }
-            }
-        }
-        ParserResultType::Array => {
-            for batch_result in reader {
-                let batch = batch_result.map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                // Filter columns if needed
-                let batch = if let Some(cols) = columns {
-                    filter_record_batch(&batch, cols)?
-                } else {
-                    batch
-                };
-
-                // Convert columnar data to rows
-                for row_idx in 0..batch.num_rows() {
-                    let mut row_vec = Vec::with_capacity(batch.num_columns());
-
-                    for column in batch.columns() {
-                        let value = extract_value_at_index(column, row_idx, strict)?;
-                        row_vec.push(value);
-                    }
-
-                    let record = RowRecord::Vec::<RandomState>(row_vec);
-                    let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-                }
-            }
-        }
-    }
-
-    Ok(())
-}
-
-/// Process Arrow IPC file with FileReader (for seekable sources)
-pub fn process_arrow_file_column_data(
-    ruby: Rc<Ruby>,
-    file: File,
-    columns: &Option<Vec<String>>,
-    result_type: ParserResultType,
-    _batch_size: Option<usize>,
-    strict: bool,
-    ruby_logger: &RubyLogger,
-) -> Result<(), ParquetGemError> {
-    let reader =
-        FileReader::try_new(file, None).map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-    let schema = reader.schema();
-    ruby_logger.debug(|| format!("Arrow file schema loaded: {:?}", schema))?;
-
-    // FileReader implements Iterator<Item = Result<RecordBatch, ArrowError>>
-    match result_type {
-        ParserResultType::Hash => {
-            let headers = OnceLock::new();
-
-            for batch_result in reader {
-                let batch = batch_result.map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                // Filter columns if needed
-                let batch = if let Some(cols) = columns {
-                    filter_record_batch(&batch, cols)?
-                } else {
-                    batch
-                };
-
-                let local_headers = headers
-                    .get_or_init(|| {
-                        let schema = batch.schema();
-                        let fields = schema.fields();
-                        let mut header_string = Vec::with_capacity(fields.len());
-                        for field in fields {
-                            header_string.push(field.name().to_owned());
-                        }
-                        StringCache::intern_many(&header_string)
-                    })
-                    .as_ref()
-                    .map_err(|e| ParquetGemError::HeaderIntern(e.clone()))?;
-
-                let mut map =
-                    HashMap::with_capacity_and_hasher(local_headers.len(), RandomState::default());
-
-                batch
-                    .columns()
-                    .iter()
-                    .enumerate()
-                    .try_for_each(|(i, column)| {
-                        let header = local_headers[i];
-                        let values = ParquetValueVec::try_from(ArrayWrapper {
-                            array: column,
-                            strict,
-                        })?;
-                        map.insert(header, values.into_inner());
-                        Ok::<_, ParquetGemError>(())
-                    })?;
-
-                let record = ColumnRecord::Map::<RandomState>(map);
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-        ParserResultType::Array => {
-            for batch_result in reader {
-                let batch = batch_result.map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))?;
-
-                // Filter columns if needed
-                let batch = if let Some(cols) = columns {
-                    filter_record_batch(&batch, cols)?
-                } else {
-                    batch
-                };
-
-                let vec = batch
-                    .columns()
-                    .iter()
-                    .map(|column| {
-                        let values = ParquetValueVec::try_from(ArrayWrapper {
-                            array: column,
-                            strict,
-                        })?;
-                        Ok::<_, ParquetGemError>(values.into_inner())
-                    })
-                    .collect::<Result<Vec<_>, _>>()?;
-
-                let record = ColumnRecord::Vec::<RandomState>(vec);
-                let _: Value = ruby.yield_value(record.try_into_value_with(&ruby)?)?;
-            }
-        }
-    }
-
-    Ok(())
-}
-
-/// Extract a single value from an Arrow array at a specific index
-fn extract_value_at_index(
-    array: &Arc<dyn arrow_array::Array>,
-    index: usize,
-    strict: bool,
-) -> Result<crate::types::ParquetField, ParquetGemError> {
-    use crate::types::ParquetField;
-    use arrow_array::*;
-    use arrow_schema::DataType;
-    use parquet::record::Field;
-
-    // Convert Arrow array value at index to Parquet Field
-    let field = match array.data_type() {
-        DataType::Boolean => {
-            let arr = array.as_any().downcast_ref::<BooleanArray>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Bool(arr.value(index))
-            }
-        }
-        DataType::Int8 => {
-            let arr = array.as_any().downcast_ref::<Int8Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Byte(arr.value(index) as i8)
-            }
-        }
-        DataType::Int16 => {
-            let arr = array.as_any().downcast_ref::<Int16Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Short(arr.value(index))
-            }
-        }
-        DataType::Int32 => {
-            let arr = array.as_any().downcast_ref::<Int32Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Int(arr.value(index))
-            }
-        }
-        DataType::Int64 => {
-            let arr = array.as_any().downcast_ref::<Int64Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Long(arr.value(index))
-            }
-        }
-        DataType::UInt8 => {
-            let arr = array.as_any().downcast_ref::<UInt8Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::UByte(arr.value(index))
-            }
-        }
-        DataType::UInt16 => {
-            let arr = array.as_any().downcast_ref::<UInt16Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::UShort(arr.value(index))
-            }
-        }
-        DataType::UInt32 => {
-            let arr = array.as_any().downcast_ref::<UInt32Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::UInt(arr.value(index))
-            }
-        }
-        DataType::UInt64 => {
-            let arr = array.as_any().downcast_ref::<UInt64Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::ULong(arr.value(index))
-            }
-        }
-        DataType::Float32 => {
-            let arr = array.as_any().downcast_ref::<Float32Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Float(arr.value(index))
-            }
-        }
-        DataType::Float64 => {
-            let arr = array.as_any().downcast_ref::<Float64Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Double(arr.value(index))
-            }
-        }
-        DataType::Utf8 => {
-            let arr = array.as_any().downcast_ref::<StringArray>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Str(arr.value(index).to_string())
-            }
-        }
-        DataType::Binary => {
-            let arr = array.as_any().downcast_ref::<BinaryArray>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Bytes(arr.value(index).into())
-            }
-        }
-        DataType::Date32 => {
-            let arr = array.as_any().downcast_ref::<Date32Array>().unwrap();
-            if arr.is_null(index) {
-                Field::Null
-            } else {
-                Field::Date(arr.value(index))
-            }
-        }
-        DataType::Timestamp(unit, _tz) => match unit {
-            arrow_schema::TimeUnit::Millisecond => {
-                let arr = array
-                    .as_any()
-                    .downcast_ref::<TimestampMillisecondArray>()
-                    .unwrap();
-                if arr.is_null(index) {
-                    Field::Null
-                } else {
-                    Field::TimestampMillis(arr.value(index))
-                }
-            }
-            arrow_schema::TimeUnit::Microsecond => {
-                let arr = array
-                    .as_any()
-                    .downcast_ref::<TimestampMicrosecondArray>()
-                    .unwrap();
-                if arr.is_null(index) {
-                    Field::Null
-                } else {
-                    Field::TimestampMicros(arr.value(index))
-                }
-            }
-            _ => Field::Null,
-        },
-        // Add more type handling as needed
-        _ => Field::Null,
-    };
-
-    // For Arrow files, we don't have Parquet logical types, so we use defaults
-    Ok(ParquetField {
-        field,
-        converted_type: parquet::basic::ConvertedType::NONE,
-        logical_type: None,
-        strict,
-    })
-}
-
-/// Filter a RecordBatch to only include specified columns
-fn filter_record_batch(
-    batch: &RecordBatch,
-    columns: &[String],
-) -> Result<RecordBatch, ParquetGemError> {
-    let schema = batch.schema();
-    let mut indices = Vec::new();
-    let mut fields = Vec::new();
-
-    for (i, field) in schema.fields().iter().enumerate() {
-        if columns.contains(&field.name().to_string()) {
-            indices.push(i);
-            fields.push(field.clone());
-        }
-    }
-
-    let new_schema = Arc::new(Schema::new(fields));
-    let new_columns: Vec<_> = indices.iter().map(|&i| batch.column(i).clone()).collect();
-
-    RecordBatch::try_new(new_schema, new_columns)
-        .map_err(|e| ParquetGemError::ArrowIpc(e.to_string()))
-}

data/ext/parquet/src/reader/format_detector.rs

@@ -1,69 +0,0 @@
-use crate::types::ParquetGemError;
-use std::io::{Read, Seek, SeekFrom};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum FileFormat {
-    Parquet,
-    Arrow,
-}
-
-/// Detect the file format by examining magic bytes
-pub fn detect_file_format<R: Read + Seek>(source: &mut R) -> Result<FileFormat, ParquetGemError> {
-    let mut magic = [0u8; 8];
-
-    // Read the first 8 bytes
-    let bytes_read = source.read(&mut magic).map_err(ParquetGemError::from)?;
-
-    // Reset to beginning
-    source
-        .seek(SeekFrom::Start(0))
-        .map_err(ParquetGemError::from)?;
-
-    if bytes_read >= 6 {
-        // Arrow IPC file format magic: "ARROW1\0\0"
-        if &magic[0..6] == b"ARROW1" {
-            return Ok(FileFormat::Arrow);
-        }
-    }
-
-    if bytes_read >= 4 {
-        // Parquet magic: "PAR1" at start
-        if &magic[0..4] == b"PAR1" {
-            return Ok(FileFormat::Parquet);
-        }
-    }
-
-    // If we can't detect from the beginning, check the end for Parquet
-    // Parquet files also have "PAR1" at the end
-    if let Ok(pos) = source.seek(SeekFrom::End(-4)) {
-        if pos >= 4 {
-            let mut end_magic = [0u8; 4];
-            if source.read_exact(&mut end_magic).is_ok() && &end_magic == b"PAR1" {
-                // Important: Reset to beginning before returning
-                source
-                    .seek(SeekFrom::Start(0))
-                    .map_err(ParquetGemError::from)?;
-                return Ok(FileFormat::Parquet);
-            }
-        }
-    }
-
-    // Always reset to beginning, even for unknown format
-    source
-        .seek(SeekFrom::Start(0))
-        .map_err(ParquetGemError::from)?;
-
-    Err(ParquetGemError::UnknownFormat)
-}
-
-/// Detect format from file extension as a fallback
-pub fn detect_format_from_extension(path: &str) -> Option<FileFormat> {
-    let lower = path.to_lowercase();
-    if lower.ends_with(".parquet") || lower.ends_with(".parq") {
-        Some(FileFormat::Parquet)
-    } else if lower.ends_with(".arrow") || lower.ends_with(".feather") || lower.ends_with(".ipc") {
-        Some(FileFormat::Arrow)
-    } else {
-        None
-    }
-}