iceberg 0.11.1 → 0.12.0

data/ext/iceberg/src/lib.rs

@@ -1,16 +1,36 @@
 mod arrow;
+mod batch;
+mod capsule;
 mod catalog;
+mod encryption;
 mod error;
+mod partitioning;
+mod result;
+mod ruby;
 mod runtime;
 mod scan;
+mod schema;
+mod snapshot;
+mod sorting;
+mod statistics;
 mod table;
 mod utils;
 
 use magnus::{Error as RbErr, Ruby, function, method, prelude::*};
 
+use crate::batch::RbArrowRecordBatch;
+use crate::capsule::RbCapsule;
 use crate::catalog::RbCatalog;
-use crate::scan::RbTableScan;
-use crate::table::RbTable;
+#[cfg(feature = "datafusion")]
+use crate::catalog::RbSessionContext;
+use crate::encryption::RbEncryptedKey;
+use crate::partitioning::{RbPartitionField, RbPartitionSpec};
+use crate::scan::{RbDataFile, RbFileScanTask, RbTableScan};
+use crate::schema::{RbNestedField, RbSchema};
+use crate::snapshot::{RbMetadataLogEntry, RbSnapshot, RbSnapshotLogEntry};
+use crate::sorting::{RbSortField, RbSortOrder};
+use crate::statistics::{RbPartitionStatisticsFile, RbStatisticsFile};
+use crate::table::{RbTable, RbTableMetadata};
 
 type RbResult<T> = Result<T, RbErr>;
 
@@ -38,93 +58,297 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("update_namespace", method!(RbCatalog::update_namespace, 2))?;
     class.define_method("drop_namespace", method!(RbCatalog::drop_namespace, 1))?;
     class.define_method("list_tables", method!(RbCatalog::list_tables, 1))?;
-    class.define_method("create_table", method!(RbCatalog::create_table, 3))?;
+    class.define_method("create_table", method!(RbCatalog::create_table, 6))?;
     class.define_method("load_table", method!(RbCatalog::load_table, 1))?;
     class.define_method("drop_table", method!(RbCatalog::drop_table, 1))?;
+    class.define_method("purge_table", method!(RbCatalog::purge_table, 1))?;
     class.define_method("table_exists?", method!(RbCatalog::table_exists, 1))?;
     class.define_method("rename_table", method!(RbCatalog::rename_table, 2))?;
     class.define_method("register_table", method!(RbCatalog::register_table, 2))?;
     #[cfg(feature = "datafusion")]
-    class.define_method("sql", method!(RbCatalog::sql, 1))?;
+    class.define_method("session_context", method!(RbCatalog::session_context, 1))?;
+
+    #[cfg(feature = "datafusion")]
+    {
+        let class = module.define_class("SessionContext", ruby.class_object())?;
+        class.define_method("sql", method!(RbSessionContext::sql, 2))?;
+    }
 
     let class = module.define_class("RbTable", ruby.class_object())?;
+    class.define_method("identifier", method!(RbTable::identifier, 0))?;
     class.define_method("scan", method!(RbTable::scan, 1))?;
     class.define_method("append", method!(RbTable::append, 2))?;
-    class.define_method("format_version", method!(RbTable::format_version, 0))?;
-    class.define_method("uuid", method!(RbTable::uuid, 0))?;
-    class.define_method("location", method!(RbTable::location, 0))?;
+    class.define_method("metadata", method!(RbTable::metadata, 0))?;
+    class.define_singleton_method(
+        "from_metadata_file",
+        function!(RbTable::from_metadata_file, 1),
+    )?;
+
+    let class = module.define_class("TableMetadata", ruby.class_object())?;
+    class.define_method(
+        "format_version",
+        method!(RbTableMetadata::format_version, 0),
+    )?;
+    class.define_method("uuid", method!(RbTableMetadata::uuid, 0))?;
+    class.define_method("location", method!(RbTableMetadata::location, 0))?;
     class.define_method(
         "last_sequence_number",
-        method!(RbTable::last_sequence_number, 0),
+        method!(RbTableMetadata::last_sequence_number, 0),
     )?;
     class.define_method(
         "next_sequence_number",
-        method!(RbTable::next_sequence_number, 0),
+        method!(RbTableMetadata::next_sequence_number, 0),
+    )?;
+    class.define_method(
+        "last_column_id",
+        method!(RbTableMetadata::last_column_id, 0),
+    )?;
+    class.define_method(
+        "last_partition_id",
+        method!(RbTableMetadata::last_partition_id, 0),
+    )?;
+    class.define_method(
+        "last_updated_ms",
+        method!(RbTableMetadata::last_updated_ms, 0),
+    )?;
+    class.define_method("schemas", method!(RbTableMetadata::schemas, 0))?;
+    class.define_method("schema_by_id", method!(RbTableMetadata::schema_by_id, 1))?;
+    class.define_method(
+        "current_schema",
+        method!(RbTableMetadata::current_schema, 0),
+    )?;
+    class.define_method(
+        "current_schema_id",
+        method!(RbTableMetadata::current_schema_id, 0),
+    )?;
+    class.define_method(
+        "partition_specs",
+        method!(RbTableMetadata::partition_specs, 0),
     )?;
-    class.define_method("last_column_id", method!(RbTable::last_column_id, 0))?;
-    class.define_method("last_partition_id", method!(RbTable::last_partition_id, 0))?;
-    class.define_method("last_updated_ms", method!(RbTable::last_updated_ms, 0))?;
-    class.define_method("schemas", method!(RbTable::schemas, 0))?;
-    class.define_method("schema_by_id", method!(RbTable::schema_by_id, 1))?;
-    class.define_method("current_schema", method!(RbTable::current_schema, 0))?;
-    class.define_method("current_schema_id", method!(RbTable::current_schema_id, 0))?;
-    class.define_method("partition_specs", method!(RbTable::partition_specs, 0))?;
     class.define_method(
         "partition_spec_by_id",
-        method!(RbTable::partition_spec_by_id, 1),
+        method!(RbTableMetadata::partition_spec_by_id, 1),
     )?;
     class.define_method(
         "default_partition_spec",
-        method!(RbTable::default_partition_spec, 0),
+        method!(RbTableMetadata::default_partition_spec, 0),
     )?;
     class.define_method(
         "default_partition_spec_id",
-        method!(RbTable::default_partition_spec_id, 0),
+        method!(RbTableMetadata::default_partition_spec_id, 0),
+    )?;
+    class.define_method("snapshots", method!(RbTableMetadata::snapshots, 0))?;
+    class.define_method(
+        "snapshot_by_id",
+        method!(RbTableMetadata::snapshot_by_id, 1),
+    )?;
+    class.define_method("history", method!(RbTableMetadata::history, 0))?;
+    class.define_method("metadata_log", method!(RbTableMetadata::metadata_log, 0))?;
+    class.define_method(
+        "current_snapshot",
+        method!(RbTableMetadata::current_snapshot, 0),
     )?;
-    class.define_method("snapshots", method!(RbTable::snapshots, 0))?;
-    class.define_method("snapshot_by_id", method!(RbTable::snapshot_by_id, 1))?;
-    class.define_method("history", method!(RbTable::history, 0))?;
-    class.define_method("metadata_log", method!(RbTable::metadata_log, 0))?;
-    class.define_method("current_snapshot", method!(RbTable::current_snapshot, 0))?;
     class.define_method(
         "current_snapshot_id",
-        method!(RbTable::current_snapshot_id, 0),
+        method!(RbTableMetadata::current_snapshot_id, 0),
+    )?;
+    class.define_method(
+        "snapshot_for_ref",
+        method!(RbTableMetadata::snapshot_for_ref, 1),
+    )?;
+    class.define_method("sort_orders", method!(RbTableMetadata::sort_orders, 0))?;
+    class.define_method(
+        "sort_order_by_id",
+        method!(RbTableMetadata::sort_order_by_id, 1),
     )?;
-    class.define_method("snapshot_for_ref", method!(RbTable::snapshot_for_ref, 1))?;
-    class.define_method("sort_orders", method!(RbTable::sort_orders, 0))?;
-    class.define_method("sort_order_by_id", method!(RbTable::sort_order_by_id, 1))?;
     class.define_method(
         "default_sort_order",
-        method!(RbTable::default_sort_order, 0),
+        method!(RbTableMetadata::default_sort_order, 0),
     )?;
     class.define_method(
         "default_sort_order_id",
-        method!(RbTable::default_sort_order_id, 0),
+        method!(RbTableMetadata::default_sort_order_id, 0),
     )?;
-    class.define_method("properties", method!(RbTable::properties, 0))?;
-    class.define_method("statistics", method!(RbTable::statistics, 0))?;
+    class.define_method("properties", method!(RbTableMetadata::properties, 0))?;
+    class.define_method("statistics", method!(RbTableMetadata::statistics, 0))?;
     class.define_method(
         "partition_statistics",
-        method!(RbTable::partition_statistics, 0),
+        method!(RbTableMetadata::partition_statistics, 0),
     )?;
     class.define_method(
         "statistics_for_snapshot",
-        method!(RbTable::statistics_for_snapshot, 1),
+        method!(RbTableMetadata::statistics_for_snapshot, 1),
     )?;
     class.define_method(
         "partition_statistics_for_snapshot",
-        method!(RbTable::partition_statistics_for_snapshot, 1),
+        method!(RbTableMetadata::partition_statistics_for_snapshot, 1),
     )?;
-    class.define_method("encryption_keys", method!(RbTable::encryption_keys, 0))?;
-    class.define_method("encryption_key", method!(RbTable::encryption_key, 1))?;
-    class.define_singleton_method(
-        "from_metadata_file",
-        function!(RbTable::from_metadata_file, 1),
+    class.define_method(
+        "encryption_keys",
+        method!(RbTableMetadata::encryption_keys, 0),
+    )?;
+    class.define_method(
+        "encryption_key",
+        method!(RbTableMetadata::encryption_key, 1),
     )?;
+    class.define_method("next_row_id", method!(RbTableMetadata::next_row_id, 0))?;
 
     let class = module.define_class("RbTableScan", ruby.class_object())?;
     class.define_method("plan_files", method!(RbTableScan::plan_files, 0))?;
     class.define_method("snapshot", method!(RbTableScan::snapshot, 0))?;
+    class.define_method("collect", method!(RbTableScan::collect, 0))?;
+    class.define_method("arrow_c_stream", method!(RbTableScan::arrow_c_stream, 0))?;
+
+    let class = module.define_class("FileScanTask", ruby.class_object())?;
+    class.define_method("file", method!(RbFileScanTask::file, 0))?;
+    class.define_method("delete_files", method!(RbFileScanTask::delete_files, 0))?;
+    class.define_method("inspect", method!(RbFileScanTask::inspect, 0))?;
+
+    let class = module.define_class("DataFile", ruby.class_object())?;
+    class.define_method("file_path", method!(RbDataFile::file_path, 0))?;
+    class.define_method("record_count", method!(RbDataFile::record_count, 0))?;
+    class.define_method(
+        "file_size_in_bytes",
+        method!(RbDataFile::file_size_in_bytes, 0),
+    )?;
+    class.define_method("equality_ids", method!(RbDataFile::equality_ids, 0))?;
+    class.define_method("inspect", method!(RbDataFile::inspect, 0))?;
+
+    let class = module.define_class("Schema", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RbSchema::new, -1))?;
+    class.define_method("fields", method!(RbSchema::fields, 0))?;
+    class.define_method("schema_id", method!(RbSchema::schema_id, 0))?;
+    class.define_method(
+        "identifier_field_ids",
+        method!(RbSchema::identifier_field_ids, 0),
+    )?;
+    class.define_method("highest_field_id", method!(RbSchema::highest_field_id, 0))?;
+    class.define_method("as_struct", method!(RbSchema::as_struct, 0))?;
+    class.define_method("arrow_c_schema", method!(RbSchema::arrow_c_schema, 0))?;
+    class.define_method("==", method!(RbSchema::eq, 1))?;
+    class.define_method("inspect", method!(RbSchema::inspect, 0))?;
+
+    let class = module.define_class("NestedField", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RbNestedField::new, 1))?;
+    class.define_method("field_id", method!(RbNestedField::field_id, 0))?;
+    class.define_method("name", method!(RbNestedField::name, 0))?;
+    class.define_method("field_type", method!(RbNestedField::field_type, 0))?;
+    class.define_method("required", method!(RbNestedField::required, 0))?;
+    class.define_method("doc", method!(RbNestedField::doc, 0))?;
+    class.define_method(
+        "initial_default",
+        method!(RbNestedField::initial_default, 0),
+    )?;
+    class.define_method("write_default", method!(RbNestedField::write_default, 0))?;
+    class.define_method("==", method!(RbNestedField::eq, 1))?;
+    class.define_method("inspect", method!(RbNestedField::inspect, 0))?;
+
+    let class = module.define_class("Capsule", ruby.class_object())?;
+    class.define_method("to_i", method!(RbCapsule::to_i, 0))?;
+    class.define_method("name", method!(RbCapsule::name, 0))?;
+
+    let class = module.define_class("ArrowRecordBatch", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RbArrowRecordBatch::new, 2))?;
+    class.define_method(
+        "arrow_c_stream",
+        method!(RbArrowRecordBatch::arrow_c_stream, 0),
+    )?;
+
+    let class = module.define_class("PartitionSpec", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RbPartitionSpec::new, -1))?;
+    class.define_method("spec_id", method!(RbPartitionSpec::spec_id, 0))?;
+    class.define_method("==", method!(RbPartitionSpec::eq, 1))?;
+    class.define_method("inspect", method!(RbPartitionSpec::inspect, 0))?;
+
+    let class = module.define_class("PartitionField", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RbPartitionField::new, 1))?;
+    class.define_method("source_id", method!(RbPartitionField::source_id, 0))?;
+    class.define_method("field_id", method!(RbPartitionField::field_id, 0))?;
+    class.define_method("name", method!(RbPartitionField::name, 0))?;
+    class.define_method("transform", method!(RbPartitionField::transform, 0))?;
+    class.define_method("==", method!(RbPartitionField::eq, 1))?;
+    class.define_method("inspect", method!(RbPartitionField::inspect, 0))?;
+
+    let class = module.define_class("SortOrder", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RbSortOrder::new, -1))?;
+    class.define_method("order_id", method!(RbSortOrder::order_id, 0))?;
+    class.define_method("fields", method!(RbSortOrder::fields, 0))?;
+    class.define_method("==", method!(RbSortOrder::eq, 1))?;
+    class.define_method("inspect", method!(RbSortOrder::inspect, 0))?;
+
+    let class = module.define_class("SortField", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RbSortField::new, 1))?;
+    class.define_method("source_id", method!(RbSortField::source_id, 0))?;
+    class.define_method("transform", method!(RbSortField::transform, 0))?;
+    class.define_method("direction", method!(RbSortField::direction, 0))?;
+    class.define_method("null_order", method!(RbSortField::null_order, 0))?;
+    class.define_method("==", method!(RbSortField::eq, 1))?;
+    class.define_method("inspect", method!(RbSortField::inspect, 0))?;
+
+    let class = module.define_class("Snapshot", ruby.class_object())?;
+    class.define_method("snapshot_id", method!(RbSnapshot::snapshot_id, 0))?;
+    class.define_method(
+        "parent_snapshot_id",
+        method!(RbSnapshot::parent_snapshot_id, 0),
+    )?;
+    class.define_method("sequence_number", method!(RbSnapshot::sequence_number, 0))?;
+    class.define_method("manifest_list", method!(RbSnapshot::manifest_list, 0))?;
+    class.define_method("schema_id", method!(RbSnapshot::schema_id, 0))?;
+    class.define_method("inspect", method!(RbSnapshot::inspect, 0))?;
+
+    let class = module.define_class("SnapshotLogEntry", ruby.class_object())?;
+    class.define_method("snapshot_id", method!(RbSnapshotLogEntry::snapshot_id, 0))?;
+    class.define_method("timestamp_ms", method!(RbSnapshotLogEntry::timestamp_ms, 0))?;
+    class.define_method("inspect", method!(RbSnapshotLogEntry::inspect, 0))?;
+
+    let class = module.define_class("MetadataLogEntry", ruby.class_object())?;
+    class.define_method(
+        "metadata_file",
+        method!(RbMetadataLogEntry::metadata_file, 0),
+    )?;
+    class.define_method("timestamp_ms", method!(RbMetadataLogEntry::timestamp_ms, 0))?;
+    class.define_method("inspect", method!(RbMetadataLogEntry::inspect, 0))?;
+
+    let class = module.define_class("EncryptedKey", ruby.class_object())?;
+    class.define_method("key_id", method!(RbEncryptedKey::key_id, 0))?;
+    class.define_method(
+        "encrypted_by_id",
+        method!(RbEncryptedKey::encrypted_by_id, 0),
+    )?;
+    class.define_method("properties", method!(RbEncryptedKey::properties, 0))?;
+    class.define_method("inspect", method!(RbEncryptedKey::inspect, 0))?;
+
+    let class = module.define_class("StatisticsFile", ruby.class_object())?;
+    class.define_method("snapshot_id", method!(RbStatisticsFile::snapshot_id, 0))?;
+    class.define_method(
+        "statistics_path",
+        method!(RbStatisticsFile::statistics_path, 0),
+    )?;
+    class.define_method(
+        "file_size_in_bytes",
+        method!(RbStatisticsFile::file_size_in_bytes, 0),
+    )?;
+    class.define_method(
+        "file_footer_size_in_bytes",
+        method!(RbStatisticsFile::file_footer_size_in_bytes, 0),
+    )?;
+    class.define_method("key_metadata", method!(RbStatisticsFile::key_metadata, 0))?;
+    class.define_method("inspect", method!(RbStatisticsFile::inspect, 0))?;
+
+    let class = module.define_class("PartitionStatisticsFile", ruby.class_object())?;
+    class.define_method(
+        "snapshot_id",
+        method!(RbPartitionStatisticsFile::snapshot_id, 0),
+    )?;
+    class.define_method(
+        "statistics_path",
+        method!(RbPartitionStatisticsFile::statistics_path, 0),
+    )?;
+    class.define_method(
+        "file_size_in_bytes",
+        method!(RbPartitionStatisticsFile::file_size_in_bytes, 0),
+    )?;
+    class.define_method("inspect", method!(RbPartitionStatisticsFile::inspect, 0))?;
 
     Ok(())
 }

data/ext/iceberg/src/partitioning.rs

@@ -0,0 +1,102 @@
+use iceberg::spec::{Transform, UnboundPartitionField, UnboundPartitionSpec};
+use magnus::{IntoValue, RArray, RHash, Ruby, TryConvert, Value, value::ReprValue};
+
+use crate::RbResult;
+use crate::error::to_rb_err;
+use crate::utils::{Wrap, rb_transform};
+
+#[magnus::wrap(class = "Iceberg::PartitionSpec")]
+pub struct RbPartitionSpec {
+    pub(crate) spec: UnboundPartitionSpec,
+}
+
+#[magnus::wrap(class = "Iceberg::PartitionField")]
+pub struct RbPartitionField {
+    pub(crate) field: UnboundPartitionField,
+}
+
+impl RbPartitionSpec {
+    pub fn new(args: &[Value]) -> RbResult<Self> {
+        let mut fields = Vec::new();
+        for v in args {
+            fields.push(<&RbPartitionField>::try_convert(*v)?.field.clone());
+        }
+        let spec = UnboundPartitionSpec::builder()
+            .with_spec_id(0)
+            .add_partition_fields(fields)
+            .map_err(to_rb_err)?
+            .build();
+        Ok(Self { spec })
+    }
+
+    pub fn spec_id(&self) -> Option<i32> {
+        self.spec.spec_id()
+    }
+
+    pub fn fields(ruby: &Ruby, rb_self: &Self) -> RArray {
+        ruby.ary_from_iter(
+            rb_self
+                .spec
+                .fields()
+                .iter()
+                .map(|v| RbPartitionField { field: v.clone() }),
+        )
+    }
+
+    pub fn eq(&self, other: &Self) -> bool {
+        self.spec == other.spec
+    }
+
+    pub fn inspect(ruby: &Ruby, rb_self: &Self) -> String {
+        format!(
+            "#<Iceberg::PartitionSpec spec_id={}, fields={}>",
+            rb_self.spec_id().into_value_with(ruby).inspect(),
+            Self::fields(ruby, rb_self).inspect(),
+        )
+    }
+}
+
+impl RbPartitionField {
+    pub fn new(ruby: &Ruby, ob: RHash) -> RbResult<Self> {
+        let field = UnboundPartitionField::builder()
+            .source_id(ob.aref(ruby.to_symbol("source_id"))?)
+            .field_id(ob.aref(ruby.to_symbol("field_id"))?)
+            .name(ob.aref(ruby.to_symbol("name"))?)
+            .transform(
+                ob.aref::<_, Wrap<Transform>>(ruby.to_symbol("transform"))?
+                    .0,
+            )
+            .build();
+        Ok(Self { field })
+    }
+
+    pub fn source_id(&self) -> i32 {
+        self.field.source_id
+    }
+
+    pub fn field_id(&self) -> Option<i32> {
+        self.field.field_id
+    }
+
+    pub fn name(&self) -> &str {
+        &self.field.name
+    }
+
+    pub fn transform(&self) -> RbResult<Value> {
+        rb_transform(&self.field.transform)
+    }
+
+    pub fn eq(&self, other: &Self) -> bool {
+        self.field == other.field
+    }
+
+    pub fn inspect(ruby: &Ruby, rb_self: &Self) -> RbResult<String> {
+        Ok(format!(
+            "#<Iceberg::PartitionField source_id={}, field_id={}, name={}, transform={}>",
+            rb_self.source_id().into_value_with(ruby).inspect(),
+            rb_self.field_id().into_value_with(ruby).inspect(),
+            rb_self.name().into_value_with(ruby).inspect(),
+            rb_self.transform()?.into_value_with(ruby).inspect(),
+        ))
+    }
+}

data/ext/iceberg/src/result.rs

@@ -0,0 +1,201 @@
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, BooleanArray, Date32Array, Decimal128Array, Float32Array, Float64Array, Int8Array,
+    Int16Array, Int32Array, Int64Array, LargeBinaryArray, StringArray, TimestampMicrosecondArray,
+    TimestampNanosecondArray, UInt8Array, UInt16Array, UInt32Array, UInt64Array,
+};
+use arrow::datatypes::{DataType as ArrowDataType, Decimal128Type};
+use arrow_array::RecordBatch;
+use arrow_array::types::DecimalType;
+use arrow_schema::TimeUnit;
+use magnus::{Class, IntoValue, Module, RArray, RClass, RModule, Ruby, Value, value::ReprValue};
+
+use crate::RbResult;
+use crate::error::todo_error;
+use crate::utils::EPOCH;
+
+pub fn collect_batches(ruby: &Ruby, batches: Vec<RecordBatch>) -> RbResult<Value> {
+    let columns = ruby.ary_new();
+    let rows = ruby.ary_new();
+
+    for batch in batches {
+        if columns.is_empty() {
+            for field in &batch.schema().fields {
+                columns.push(ruby.str_new(field.name()))?;
+            }
+        }
+
+        for _ in 0..batch.num_rows() {
+            rows.push(ruby.ary_new())?;
+        }
+
+        for column in batch.columns() {
+            match column.data_type() {
+                ArrowDataType::Null => collect_column_null(ruby, column, rows)?,
+                ArrowDataType::Boolean => collect_column_boolean(ruby, column, rows)?,
+                ArrowDataType::Int8 => collect_column_int8(ruby, column, rows)?,
+                ArrowDataType::Int16 => collect_column_int16(ruby, column, rows)?,
+                ArrowDataType::Int32 => collect_column_int32(ruby, column, rows)?,
+                ArrowDataType::Int64 => collect_column_int64(ruby, column, rows)?,
+                ArrowDataType::UInt8 => collect_column_uint8(ruby, column, rows)?,
+                ArrowDataType::UInt16 => collect_column_uint16(ruby, column, rows)?,
+                ArrowDataType::UInt32 => collect_column_uint32(ruby, column, rows)?,
+                ArrowDataType::UInt64 => collect_column_uint64(ruby, column, rows)?,
+                ArrowDataType::Float32 => collect_column_float32(ruby, column, rows)?,
+                ArrowDataType::Float64 => collect_column_float64(ruby, column, rows)?,
+                ArrowDataType::Decimal128(precision, scale) => {
+                    collect_column_decimal128(ruby, column, rows, *precision, *scale)?
+                }
+                ArrowDataType::Date32 => collect_column_date32(ruby, column, rows)?,
+                ArrowDataType::Timestamp(TimeUnit::Microsecond, None) => {
+                    collect_column_timestamp_us(ruby, column, rows)?
+                }
+                ArrowDataType::Timestamp(TimeUnit::Nanosecond, None) => {
+                    collect_column_timestamp_ns(ruby, column, rows)?
+                }
+                ArrowDataType::Utf8 => collect_column_utf8(ruby, column, rows)?,
+                ArrowDataType::LargeBinary => collect_column_large_binary(ruby, column, rows)?,
+                _ => return Err(todo_error(column.data_type())),
+            }
+        }
+    }
+
+    ruby.class_object()
+        .const_get::<_, RModule>("Iceberg")?
+        .const_get::<_, RClass>("Result")?
+        .new_instance((columns, rows))
+}
+
+pub fn collect_column_null(ruby: &Ruby, column: &Arc<dyn Array>, rows: RArray) -> RbResult<()> {
+    for i in 0..column.len() {
+        rows.entry::<RArray>(i.try_into().unwrap())?
+            .push(ruby.qnil())?;
+    }
+    Ok(())
+}
+
+macro_rules! collect_column {
+    ($name:ident, $type:ty) => {
+        pub fn $name(ruby: &Ruby, column: &Arc<dyn Array>, rows: RArray) -> RbResult<()> {
+            let array = column.as_any().downcast_ref::<$type>().unwrap();
+            for (i, value) in array.iter().enumerate() {
+                rows.entry::<RArray>(i.try_into().unwrap())?
+                    .push(value.map(|v| v.into_value_with(ruby)))?;
+            }
+            Ok(())
+        }
+    };
+}
+
+collect_column!(collect_column_boolean, BooleanArray);
+collect_column!(collect_column_int8, Int8Array);
+collect_column!(collect_column_int16, Int16Array);
+collect_column!(collect_column_int32, Int32Array);
+collect_column!(collect_column_int64, Int64Array);
+collect_column!(collect_column_uint8, UInt8Array);
+collect_column!(collect_column_uint16, UInt16Array);
+collect_column!(collect_column_uint32, UInt32Array);
+collect_column!(collect_column_uint64, UInt64Array);
+collect_column!(collect_column_float32, Float32Array);
+collect_column!(collect_column_float64, Float64Array);
+collect_column!(collect_column_utf8, StringArray);
+
+pub fn collect_column_decimal128(
+    ruby: &Ruby,
+    column: &Arc<dyn Array>,
+    rows: RArray,
+    precision: u8,
+    scale: i8,
+) -> RbResult<()> {
+    ruby.require("bigdecimal")?;
+    let array = column.as_any().downcast_ref::<Decimal128Array>().unwrap();
+    for (i, value) in array.iter().enumerate() {
+        let value: Option<Value> = value
+            .map(|v| {
+                ruby.class_object().funcall(
+                    "BigDecimal",
+                    (Decimal128Type::format_decimal(v, precision, scale).into_value_with(ruby),),
+                )
+            })
+            .transpose()?;
+        rows.entry::<RArray>(i.try_into().unwrap())?.push(value)?;
+    }
+    Ok(())
+}
+
+pub fn collect_column_date32(ruby: &Ruby, column: &Arc<dyn Array>, rows: RArray) -> RbResult<()> {
+    let epoch = ruby.get_inner(&EPOCH);
+    let array = column.as_any().downcast_ref::<Date32Array>().unwrap();
+    for (i, value) in array.iter().enumerate() {
+        rows.entry::<RArray>(i.try_into().unwrap())?.push(
+            value
+                .map(|v| epoch.funcall::<_, _, Value>("+", (v,)))
+                .transpose()?,
+        )?;
+    }
+    Ok(())
+}
+
+pub fn collect_column_timestamp_us(
+    ruby: &Ruby,
+    column: &Arc<dyn Array>,
+    rows: RArray,
+) -> RbResult<()> {
+    let time_class = ruby.class_object().const_get::<_, Value>("Time")?;
+    let time_unit = ruby.to_symbol("usec");
+    let array = column
+        .as_any()
+        .downcast_ref::<TimestampMicrosecondArray>()
+        .unwrap();
+    for (i, value) in array.iter().enumerate() {
+        let value: Option<Value> = match value {
+            Some(v) => {
+                let sec = v / 1_000_000;
+                let usec = v % 1_000_000;
+                Some(time_class.funcall("at", (sec, usec, time_unit))?)
+            }
+            None => None,
+        };
+        rows.entry::<RArray>(i.try_into().unwrap())?.push(value)?;
+    }
+    Ok(())
+}
+
+pub fn collect_column_timestamp_ns(
+    ruby: &Ruby,
+    column: &Arc<dyn Array>,
+    rows: RArray,
+) -> RbResult<()> {
+    let time_class = ruby.class_object().const_get::<_, Value>("Time")?;
+    let time_unit = ruby.to_symbol("nsec");
+    let array = column
+        .as_any()
+        .downcast_ref::<TimestampNanosecondArray>()
+        .unwrap();
+    for (i, value) in array.iter().enumerate() {
+        let value: Option<Value> = match value {
+            Some(v) => {
+                let sec = v / 1_000_000_000;
+                let nsec = v % 1_000_000_000;
+                Some(time_class.funcall("at", (sec, nsec, time_unit))?)
+            }
+            None => None,
+        };
+        rows.entry::<RArray>(i.try_into().unwrap())?.push(value)?;
+    }
+    Ok(())
+}
+
+pub fn collect_column_large_binary(
+    ruby: &Ruby,
+    column: &Arc<dyn Array>,
+    rows: RArray,
+) -> RbResult<()> {
+    let array = column.as_any().downcast_ref::<LargeBinaryArray>().unwrap();
+    for (i, value) in array.iter().enumerate() {
+        rows.entry::<RArray>(i.try_into().unwrap())?
+            .push(value.map(|v| ruby.str_from_slice(v)))?;
+    }
+    Ok(())
+}