parquet 0.2.12-arm64-darwin → 0.3.0-arm64-darwin

data/ext/parquet/src/ruby_integration.rs

@@ -1,77 +0,0 @@
-use std::{
-    fs::File,
-    io::{BufReader, SeekFrom},
-    mem::ManuallyDrop,
-};
-
-use bytes::Bytes;
-use magnus::{value::Opaque, Ruby, Value};
-use parquet::{
-    errors::ParquetError,
-    file::reader::{ChunkReader, Length},
-};
-use std::io::Read;
-
-use crate::ruby_reader::{build_ruby_reader, SeekableRead};
-
-const READ_BUFFER_SIZE: usize = 16 * 1024;
-
-pub struct SeekableRubyValue(pub Opaque<Value>);
-
-impl Length for SeekableRubyValue {
-    fn len(&self) -> u64 {
-        let ruby = unsafe { Ruby::get_unchecked() };
-        let mut reader = build_ruby_reader(&ruby, ruby.get_inner(self.0)).unwrap();
-        let current_pos = reader.seek(SeekFrom::Current(0)).unwrap();
-        let file_len = reader.seek(SeekFrom::End(0)).unwrap();
-        reader.seek(SeekFrom::Start(current_pos)).unwrap();
-        file_len
-    }
-}
-
-impl ChunkReader for SeekableRubyValue {
-    type T = BufReader<Box<dyn SeekableRead>>;
-
-    fn get_read(&self, start: u64) -> parquet::errors::Result<Self::T> {
-        let ruby = unsafe { Ruby::get_unchecked() };
-        let mut reader = build_ruby_reader(&ruby, ruby.get_inner(self.0)).unwrap();
-        reader.seek(SeekFrom::Start(start))?;
-        Ok(BufReader::with_capacity(READ_BUFFER_SIZE, reader))
-    }
-
-    fn get_bytes(&self, start: u64, length: usize) -> parquet::errors::Result<Bytes> {
-        let ruby = unsafe { Ruby::get_unchecked() };
-        let mut buffer = Vec::with_capacity(length);
-        let mut reader = build_ruby_reader(&ruby, ruby.get_inner(self.0)).unwrap();
-        reader.seek(SeekFrom::Start(start))?;
-        let read = reader.take(length as _).read_to_end(&mut buffer)?;
-
-        if read != length {
-            return Err(ParquetError::EOF(format!(
-                "Expected to read {} bytes, read only {}",
-                length, read
-            )));
-        }
-        Ok(buffer.into())
-    }
-}
-
-pub struct ForgottenFileHandle(pub ManuallyDrop<File>);
-
-impl Length for ForgottenFileHandle {
-    fn len(&self) -> u64 {
-        self.0.len()
-    }
-}
-
-impl ChunkReader for ForgottenFileHandle {
-    type T = BufReader<File>;
-
-    fn get_read(&self, start: u64) -> parquet::errors::Result<Self::T> {
-        self.0.get_read(start)
-    }
-
-    fn get_bytes(&self, start: u64, length: usize) -> parquet::errors::Result<Bytes> {
-        self.0.get_bytes(start, length)
-    }
-}

data/ext/parquet/src/ruby_reader.rs

@@ -1,171 +0,0 @@
-use magnus::{
-    value::{Opaque, ReprValue},
-    RClass, RString, Ruby, Value,
-};
-use std::io::{self, Read, Seek, SeekFrom, Write};
-use std::sync::OnceLock;
-
-static STRING_IO_CLASS: OnceLock<Opaque<RClass>> = OnceLock::new();
-
-/// A reader that can handle various Ruby input types (String, StringIO, IO-like objects)
-/// and provide a standard Read implementation for them.
-pub struct RubyReader<T> {
-    inner: T,
-    offset: usize,
-}
-
-pub trait SeekableRead: Read + Seek {}
-impl<T: Read + Seek> SeekableRead for T {}
-
-pub fn build_ruby_reader(
-    ruby: &Ruby,
-    input: Value,
-) -> Result<Box<dyn SeekableRead>, magnus::Error> {
-    if RubyReader::is_string_io(ruby, &input) {
-        RubyReader::from_string_io(ruby, input)
-    } else if RubyReader::is_io_like(&input) {
-        RubyReader::from_io(input)
-    } else {
-        RubyReader::from_string_like(input)
-    }
-}
-
-impl Seek for RubyReader<Value> {
-    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
-        let (whence, offset) = match pos {
-            SeekFrom::Start(i) => (0, i as i64),
-            SeekFrom::Current(i) => (1, i),
-            SeekFrom::End(i) => (2, i),
-        };
-
-        let new_position = self
-            .inner
-            .funcall("seek", (offset, whence))
-            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
-
-        Ok(new_position)
-    }
-}
-
-impl Write for RubyReader<Value> {
-    fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {
-        let ruby_bytes = RString::from_slice(buf);
-
-        let bytes_written = self
-            .inner
-            .funcall::<_, _, usize>("write", (ruby_bytes,))
-            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
-
-        Ok(bytes_written)
-    }
-
-    fn flush(&mut self) -> Result<(), io::Error> {
-        self.inner
-            .funcall::<_, _, Value>("flush", ())
-            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
-
-        Ok(())
-    }
-}
-
-impl Seek for RubyReader<RString> {
-    fn seek(&mut self, pos: io::SeekFrom) -> io::Result<u64> {
-        match pos {
-            io::SeekFrom::Start(offset) => self.offset = offset as usize,
-            io::SeekFrom::Current(offset) => self.offset = (self.offset as i64 + offset) as usize,
-            io::SeekFrom::End(offset) => self.offset = self.inner.len() - offset as usize,
-        }
-        Ok(self.offset as u64)
-    }
-}
-
-impl RubyReader<Value> {
-    fn from_io(input: Value) -> Result<Box<dyn SeekableRead>, magnus::Error> {
-        if Self::is_io_like(&input) {
-            Ok(Box::new(Self::from_io_like(input)))
-        } else {
-            Err(magnus::Error::new(
-                magnus::exception::type_error(),
-                "Input is not an IO-like object",
-            ))
-        }
-    }
-
-    fn is_io_like(input: &Value) -> bool {
-        input.respond_to("read", false).unwrap_or(false)
-    }
-
-    fn from_io_like(input: Value) -> Self {
-        Self {
-            inner: input,
-            offset: 0,
-        }
-    }
-}
-
-impl RubyReader<RString> {
-    pub fn from_string_io(
-        ruby: &Ruby,
-        input: Value,
-    ) -> Result<Box<dyn SeekableRead>, magnus::Error> {
-        if !Self::is_string_io(ruby, &input) {
-            return Err(magnus::Error::new(
-                magnus::exception::type_error(),
-                "Input is not a StringIO",
-            ));
-        }
-
-        let string_content = input.funcall::<_, _, RString>("string", ()).unwrap();
-        Ok(Box::new(Self {
-            inner: string_content,
-            offset: 0,
-        }))
-    }
-
-    fn is_string_io(ruby: &Ruby, input: &Value) -> bool {
-        let string_io_class = STRING_IO_CLASS.get_or_init(|| {
-            let class = RClass::from_value(ruby.eval("StringIO").unwrap()).unwrap();
-            Opaque::from(class)
-        });
-        input.is_kind_of(ruby.get_inner(*string_io_class))
-    }
-
-    fn from_string_like(input: Value) -> Result<Box<dyn SeekableRead>, magnus::Error> {
-        // Try calling `to_str`, and if that fails, try `to_s`
-        let string_content = input
-            .funcall::<_, _, RString>("to_str", ())
-            .or_else(|_| input.funcall::<_, _, RString>("to_s", ()))?;
-        Ok(Box::new(Self {
-            inner: string_content,
-            offset: 0,
-        }))
-    }
-}
-
-impl Read for RubyReader<Value> {
-    fn read(&mut self, mut buf: &mut [u8]) -> io::Result<usize> {
-        let bytes = self
-            .inner
-            .funcall::<_, _, RString>("read", (buf.len(),))
-            .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?;
-
-        buf.write_all(unsafe { bytes.as_slice() })?;
-
-        Ok(bytes.len())
-    }
-}
-
-impl Read for RubyReader<RString> {
-    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-        let string_buffer = unsafe { self.inner.as_slice() };
-        if self.offset >= string_buffer.len() {
-            return Ok(0); // EOF
-        }
-
-        let remaining = string_buffer.len() - self.offset;
-        let copy_size = remaining.min(buf.len());
-        buf[..copy_size].copy_from_slice(&string_buffer[self.offset..self.offset + copy_size]);
-        self.offset += copy_size;
-        Ok(copy_size)
-    }
-}

data/ext/parquet/src/types/core_types.rs

@@ -1,75 +0,0 @@
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum ParserResultType {
-    Hash,
-    Array,
-}
-
-impl ParserResultType {
-    pub fn iter() -> impl Iterator<Item = Self> {
-        [Self::Hash, Self::Array].into_iter()
-    }
-}
-
-impl TryFrom<&str> for ParserResultType {
-    type Error = String;
-
-    fn try_from(value: &str) -> Result<Self, Self::Error> {
-        match value {
-            "hash" => Ok(ParserResultType::Hash),
-            "array" => Ok(ParserResultType::Array),
-            _ => Err(format!("Invalid parser result type: {}", value)),
-        }
-    }
-}
-
-impl TryFrom<String> for ParserResultType {
-    type Error = String;
-
-    fn try_from(value: String) -> Result<Self, Self::Error> {
-        Self::try_from(value.as_str())
-    }
-}
-
-impl std::fmt::Display for ParserResultType {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            ParserResultType::Hash => write!(f, "hash"),
-            ParserResultType::Array => write!(f, "array"),
-        }
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct ListField<'a> {
-    pub item_type: ParquetSchemaType<'a>,
-    pub format: Option<&'a str>,
-}
-
-#[derive(Debug, Clone)]
-pub struct MapField<'a> {
-    pub key_type: ParquetSchemaType<'a>,
-    pub value_type: ParquetSchemaType<'a>,
-    pub format: Option<&'a str>,
-}
-
-#[derive(Debug, Clone)]
-pub enum ParquetSchemaType<'a> {
-    Int8,
-    Int16,
-    Int32,
-    Int64,
-    UInt8,
-    UInt16,
-    UInt32,
-    UInt64,
-    Float,
-    Double,
-    String,
-    Binary,
-    Boolean,
-    Date32,
-    TimestampMillis,
-    TimestampMicros,
-    List(Box<ListField<'a>>),
-    Map(Box<MapField<'a>>),
-}

data/ext/parquet/src/types/mod.rs

@@ -1,30 +0,0 @@
-// Re-export all public items from submodules
-mod core_types;
-mod parquet_value;
-mod record_types;
-mod timestamp;
-mod type_conversion;
-mod writer_types;
-
-pub use core_types::*;
-pub use parquet_value::*;
-pub use record_types::*;
-pub use timestamp::*;
-pub use type_conversion::*;
-pub use writer_types::*;
-
-// Common imports used across the module
-use arrow_array::cast::downcast_array;
-use arrow_array::{
-    Array, BinaryArray, BooleanArray, Date32Array, Date64Array, Float16Array, Float32Array,
-    Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, ListArray, NullArray, StringArray,
-    StructArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
-    TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
-};
-use arrow_schema::{DataType, TimeUnit};
-use magnus::{value::ReprValue, Error as MagnusError, IntoValue, Ruby, TryConvert, Value};
-use parquet::data_type::Decimal;
-use parquet::record::Field;
-use std::{collections::HashMap, hash::BuildHasher, sync::Arc};
-
-use crate::header_cache::StringCacheKey;