parquet 0.0.5 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 398a8ab4fe6b9c8e82d63ec832aa73163e75874c39080d87291a60397756df42
-  data.tar.gz: cace20e14d0eddc6e3185b2f9294253cb57c1689ec463ff66bc903d3c780af13
+  metadata.gz: 90e876ca198a0e1871f692a382f09ceaeec670d162da26f2c102ea4eca4244bf
+  data.tar.gz: 96743e260cbd2fb55f6cdeaf256fbb1e915c57651fdc3f20fdd58b6a34596544
 SHA512:
-  metadata.gz: 72ae6542b367fe433016f06fa109aaa77fe360bbc1df64e5c997db8fcc0a00aa166aa19a37240a706b3f443612770b80bc387dd41b34ee4a94ab26c3b0e74832
-  data.tar.gz: f69b10c6d4c8d879cdd3fce7c3b44933a99569358d1adfa3106760bd7c66036a2fef86737cf4dc6369be46234c124b9f2ef66e82fab118e36b5b079e9d23e10b
+  metadata.gz: 1609a37c5a9bd9f1d57bb31dd02b2fdb5b608a7c044686e6ef2513c95e53e830bd7bf7048a36904465a32a5915425c7b6bf581c5b35a4fb19f950cbca20913b2
+  data.tar.gz: 96ec18377fc5944556760329c126f440de61d3b378bfa976a66437db03f0a51220c880afd14098a5b1968daa968d2e836c50f83bef21507789ba4df314c48148

data/Cargo.lock

@@ -387,6 +387,22 @@ version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
+[[package]]
+name = "errno"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+
 [[package]]
 name = "flatbuffers"
 version = "24.12.23"
@@ -934,6 +950,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
+
 [[package]]
 name = "litemap"
 version = "0.7.4"
@@ -1185,6 +1207,7 @@ dependencies = [
  "mimalloc",
  "parquet 54.0.0",
  "rb-sys",
+ "tempfile",
  "thiserror",
 ]
 
@@ -1377,6 +1400,19 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustix"
+version = "0.38.43"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6"
+dependencies = [
+ "bitflags 2.6.0",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.18"
@@ -1530,6 +1566,20 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "getrandom",
+ "once_cell",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "thiserror"
 version = "2.0.9"

data/README.md

@@ -4,8 +4,6 @@
 
 This project is a Ruby library wrapping the [parquet-rs](https://github.com/apache/parquet-rs) rust crate.
 
-At the moment, it only supports iterating rows as either a hash or an array.
-
 ## Usage
 
 This library provides high-level bindings to parquet-rs with two primary APIs for reading Parquet files: row-wise and column-wise iteration. The column-wise API generally offers better performance, especially when working with subset of columns.
@@ -83,3 +81,95 @@ Additional arguments for `each_column`:
 - `batch_size`: Number of rows per batch (defaults to implementation-defined value)
 
 When no block is given, both methods return an Enumerator.
+
+### Writing Row-wise Data
+
+The `write_rows` method allows you to write data row by row:
+
+```ruby
+require "parquet"
+
+# Define the schema for your data
+schema = [
+  { "id" => "int64" },
+  { "name" => "string" },
+  { "score" => "double" }
+]
+
+# Create an enumerator that yields arrays of row values
+rows = [
+  [1, "Alice", 95.5],
+  [2, "Bob", 82.3],
+  [3, "Charlie", 88.7]
+].each
+
+# Write to a file
+Parquet.write_rows(rows, schema: schema, write_to: "data.parquet")
+
+# Write to an IO object
+File.open("data.parquet", "wb") do |file|
+  Parquet.write_rows(rows, schema: schema, write_to: file)
+end
+
+# Optionally specify batch size (default is 1000)
+Parquet.write_rows(rows,
+  schema: schema,
+  write_to: "data.parquet",
+  batch_size: 500
+)
+```
+
+### Writing Column-wise Data
+
+The `write_columns` method provides a more efficient way to write data in column-oriented batches:
+
+```ruby
+require "parquet"
+
+# Define the schema
+schema = [
+  { "id" => "int64" },
+  { "name" => "string" },
+  { "score" => "double" }
+]
+
+# Create batches of column data
+batches = [
+  # First batch
+  [
+    [1, 2],          # id column
+    ["Alice", "Bob"], # name column
+    [95.5, 82.3]     # score column
+  ],
+  # Second batch
+  [
+    [3],             # id column
+    ["Charlie"],     # name column
+    [88.7]           # score column
+  ]
+]
+
+# Create an enumerator from the batches
+columns = batches.each
+
+# Write to a parquet file
+Parquet.write_columns(columns, schema: schema, write_to: "data.parquet")
+
+# Write to an IO object
+File.open("data.parquet", "wb") do |file|
+  Parquet.write_columns(columns, schema: schema, write_to: file)
+end
+```
+
+The following data types are supported in the schema:
+
+- `int8`, `int16`, `int32`, `int64`
+- `uint8`, `uint16`, `uint32`, `uint64`
+- `float`, `double`
+- `string`
+- `binary`
+- `boolean`
+- `date32`
+- `timestamp_millis`, `timestamp_micros`
+
+Note: List and Map types are currently not supported.

data/ext/parquet/Cargo.toml

@@ -17,6 +17,7 @@ magnus = { version = "0.7", features = ["rb-sys"] }
 parquet = { version = "^54.0", features = ["json", "object_store"] }
 rb-sys = "^0.9"
 thiserror = "2.0"
+tempfile = "^3.15"
 
 [target.'cfg(target_os = "linux")'.dependencies]
 jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }

data/ext/parquet/src/lib.rs

@@ -6,9 +6,7 @@ mod ruby_integration;
 mod ruby_reader;
 mod types;
 mod utils;
-
-mod parquet_column_reader;
-mod parquet_row_reader;
+mod writer;
 
 use crate::enumerator::*;
 use crate::reader::*;
@@ -16,6 +14,8 @@ use crate::ruby_integration::*;
 use crate::types::*;
 
 use magnus::{Error, Ruby};
+use writer::write_columns;
+use writer::write_rows;
 
 /// Initializes the Ruby extension and defines methods.
 #[magnus::init]
@@ -23,5 +23,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = ruby.define_module("Parquet")?;
     module.define_module_function("each_row", magnus::method!(parse_parquet_rows, -1))?;
     module.define_module_function("each_column", magnus::method!(parse_parquet_columns, -1))?;
+    module.define_module_function("write_rows", magnus::function!(write_rows, -1))?;
+    module.define_module_function("write_columns", magnus::function!(write_columns, -1))?;
     Ok(())
 }

data/ext/parquet/src/{reader.rs → reader/mod.rs}

@@ -1,11 +1,14 @@
+mod parquet_column_reader;
+mod parquet_row_reader;
+
 use std::io;
 
 use magnus::{Error as MagnusError, Ruby};
 use thiserror::Error;
 
 use crate::header_cache::CacheError;
-pub use crate::parquet_column_reader::parse_parquet_columns;
-pub use crate::parquet_row_reader::parse_parquet_rows;
+pub use parquet_column_reader::parse_parquet_columns;
+pub use parquet_row_reader::parse_parquet_rows;
 
 #[derive(Error, Debug)]
 pub enum ReaderError {

data/ext/parquet/src/types/core_types.rs

@@ -0,0 +1,73 @@
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum ParserResultType {
+    Hash,
+    Array,
+}
+
+impl ParserResultType {
+    pub fn iter() -> impl Iterator<Item = Self> {
+        [Self::Hash, Self::Array].into_iter()
+    }
+}
+
+impl TryFrom<&str> for ParserResultType {
+    type Error = String;
+
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        match value {
+            "hash" => Ok(ParserResultType::Hash),
+            "array" => Ok(ParserResultType::Array),
+            _ => Err(format!("Invalid parser result type: {}", value)),
+        }
+    }
+}
+
+impl TryFrom<String> for ParserResultType {
+    type Error = String;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        Self::try_from(value.as_str())
+    }
+}
+
+impl std::fmt::Display for ParserResultType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ParserResultType::Hash => write!(f, "hash"),
+            ParserResultType::Array => write!(f, "array"),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct ListField {
+    pub item_type: ParquetSchemaType,
+}
+
+#[derive(Debug, Clone)]
+pub struct MapField {
+    pub key_type: ParquetSchemaType,
+    pub value_type: ParquetSchemaType,
+}
+
+#[derive(Debug, Clone)]
+pub enum ParquetSchemaType {
+    Int8,
+    Int16,
+    Int32,
+    Int64,
+    UInt8,
+    UInt16,
+    UInt32,
+    UInt64,
+    Float,
+    Double,
+    String,
+    Binary,
+    Boolean,
+    Date32,
+    TimestampMillis,
+    TimestampMicros,
+    List(Box<ListField>),
+    Map(Box<MapField>),
+}

data/ext/parquet/src/types/mod.rs

@@ -0,0 +1,30 @@
+// Re-export all public items from submodules
+mod core_types;
+mod parquet_value;
+mod record_types;
+mod timestamp;
+mod type_conversion;
+mod writer_types;
+
+pub use core_types::*;
+pub use parquet_value::*;
+pub use record_types::*;
+pub use timestamp::*;
+pub use type_conversion::*;
+pub use writer_types::*;
+
+// Common imports used across the module
+use arrow_array::cast::downcast_array;
+use arrow_array::{
+    Array, BinaryArray, BooleanArray, Date32Array, Date64Array, Float16Array, Float32Array,
+    Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, ListArray, NullArray, StringArray,
+    StructArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
+    TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+};
+use arrow_schema::{DataType, TimeUnit};
+use magnus::{value::ReprValue, Error as MagnusError, IntoValue, Ruby, TryConvert, Value};
+use parquet::data_type::Decimal;
+use parquet::record::Field;
+use std::{collections::HashMap, hash::BuildHasher, sync::Arc};
+
+use crate::header_cache::StringCacheKey;