iceberg 0.11.2 → 0.12.1

data/Cargo.toml

@@ -2,5 +2,12 @@
 members = ["ext/iceberg"]
 resolver = "3"
 
+[profile.dev]
+strip = true
+opt-level = 2 # compiles faster
+
 [profile.release]
 strip = true
+opt-level = "s"
+lto = true
+codegen-units = 1

data/NOTICE.txt

@@ -1,6 +1,6 @@
 
 Apache Iceberg
-Copyright 2017-2025 The Apache Software Foundation
+Copyright 2017-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
@@ -8,7 +8,7 @@ The Apache Software Foundation (http://www.apache.org/).
 --------------------------------------------------------------------------------
 
 Apache Iceberg Rust
-Copyright 2023-2024 The Apache Software Foundation
+Copyright 2023-2026 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

data/README.md

@@ -19,10 +19,10 @@ gem "iceberg"
 Create a client for an Iceberg catalog
 
 ```ruby
-catalog = Iceberg::RestCatalog.new(uri: "http://localhost:8181")
+catalog = Iceberg::RestCatalog.new(uri: "http://localhost:8181", default_namespace: "main")
 ```
 
-Create a namespace
+Create a namespace if needed
 
 ```ruby
 catalog.create_namespace("main")
@@ -31,30 +31,53 @@ catalog.create_namespace("main")
 Create a table
 
 ```ruby
-catalog.create_table("main.events") do |t|
-  t.bigint "id"
-  t.float "value"
-end
+table =
+  catalog.create_table("events") do |t|
+    t.bigint "id"
+    t.float "value"
+  end
+```
+
+Append data
+
+```ruby
+table.append([{"id" => 1, "value" => 3.0}, {"id" => 2, "value" => 4.0}])
 ```
 
 Or with [Polars](https://github.com/ankane/ruby-polars)
 
 ```ruby
 df = Polars::DataFrame.new({"id" => [1, 2], "value" => [3.0, 4.0]})
-table = catalog.create_table("main.events", schema: df.schema)
+table = catalog.create_table("events", schema: df.schema)
 table.append(df)
 ```
 
+Or with [Nanoarrow](https://github.com/ankane/nanoarrow-ruby)
+
+```ruby
+arr =
+  Nanoarrow::Array.new(
+    [{"id" => 1, "value" => 3.0}, {"id" => 2, "value" => 4.0}],
+    Nanoarrow.struct({"a" => Nanoarrow.int64, "b" => Nanoarrow.string})
+  )
+table = catalog.create_table("events", schema: arr.schema)
+table.append(arr)
+```
+
 Load a table
 
 ```ruby
-table = catalog.load_table("main.events")
+table = catalog.load_table("events")
 ```
 
 Query a table
 
 ```ruby
+table.to_a
+# or
 table.to_polars.collect
+# or
+table.scan.to_arrow
 ```
 
 ## Catalog Types
@@ -67,6 +90,22 @@ Iceberg::RestCatalog.new(
 )
 ```
 
+### S3 Tables
+
+```ruby
+Iceberg::S3TablesCatalog.new(
+  arn: "arn:aws:s3tables:..."
+)
+```
+
+### Glue
+
+```ruby
+Iceberg::GlueCatalog.new(
+  warehouse: "s3://my-bucket"
+)
+```
+
 ### SQL
 
 ```ruby
@@ -133,7 +172,7 @@ catalog.list_tables("main")
 Create a table
 
 ```ruby
-catalog.create_table("main.events") do |t|
+catalog.create_table("events") do |t|
   t.integer "id"
   t.float "value"
 end
@@ -142,31 +181,31 @@ end
 Load a table
 
 ```ruby
-catalog.load_table("main.events")
+catalog.load_table("events")
 ```
 
 Check if a table exists
 
 ```ruby
-catalog.table_exists?("main.events")
+catalog.table_exists?("events")
 ```
 
 Rename a table
 
 ```ruby
-catalog.rename_table("main.events", "main.events2")
+catalog.rename_table("events", "events2")
 ```
 
 Register a table
 
 ```ruby
-catalog.register_table("main.events", "metadata.json")
+catalog.register_table("events", "metadata.json")
 ```
 
 Drop a table
 
 ```ruby
-catalog.drop_table("main.events")
+catalog.drop_table("events")
 ```
 
 ## Static Tables
@@ -196,7 +235,7 @@ To get started with development:
 git clone https://github.com/ankane/iceberg-ruby.git
 cd iceberg-ruby
 bundle install
-bundle exec rake compile
+bundle exec rake compile:dev
 
 # memory catalog
 bundle exec rake test:memory
@@ -208,4 +247,10 @@ bundle exec rake test:rest
 # SQL catalog
 createdb iceberg_ruby_test
 bundle exec rake test:sql
+
+# S3 Tables catalog
+bundle exec rake test:s3tables
+
+# Glue catalog
+bundle exec rake test:glue
 ```

data/ext/iceberg/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "iceberg-ruby"
-version = "0.11.2"
+version = "0.12.1"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2024"
-rust-version = "1.92"
+rust-version = "1.95"
 publish = false
 
 [lib]
@@ -12,25 +12,28 @@ name = "iceberg"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow-array = { version = "57", features = ["ffi"] }
-arrow-schema = "57"
-datafusion = { version = "52", optional = true }
+arrow = "58.4"
+arrow-array = { version = "58.4", features = ["ffi"] }
+arrow-cast = "58.4"
+arrow-schema = "58.4"
+datafusion = { version = "54", optional = true }
 futures = "0.3"
-iceberg = "=0.9.1"
-iceberg-catalog-glue = { version = "=0.9.1", optional = true }
-iceberg-catalog-rest = { version = "=0.9.1", optional = true }
-iceberg-catalog-s3tables = { version = "=0.9.1", optional = true }
-iceberg-catalog-sql = { version = "=0.9.1", optional = true }
-iceberg-datafusion = { version = "=0.9.1", optional = true }
-magnus = "0.8"
-parquet = "57"
+iceberg = "=0.11.0"
+iceberg-catalog-glue = { version = "=0.11.0", optional = true }
+iceberg-catalog-rest = { version = "=0.11.0", optional = true }
+iceberg-catalog-s3tables = { version = "=0.11.0", optional = true }
+iceberg-catalog-sql = { version = "=0.11.0", optional = true }
+iceberg-datafusion = { version = "=0.11.0", optional = true }
+iceberg-storage-opendal = "=0.11.0"
+magnus = "0.9"
+parquet = "58.4"
 rb-sys = "0.9"
-sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite"], default-features = false, optional = true }
+sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite", "tls-rustls-aws-lc-rs"], default-features = false, optional = true }
 tokio = { version = "1", features = ["rt-multi-thread"] }
 uuid = { version = "1", features = ["v4"] }
 
 [features]
-default = ["rest", "sql"]
+default = ["glue", "rest", "s3tables", "sql"]
 datafusion = ["dep:datafusion", "dep:iceberg-datafusion"]
 glue = ["dep:iceberg-catalog-glue"]
 rest = ["dep:iceberg-catalog-rest"]

data/ext/iceberg/src/arrow.rs

@@ -1,11 +1,12 @@
 use arrow_array::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
+use arrow_schema::Schema as ArrowSchema;
+use arrow_schema::ffi::FFI_ArrowSchema;
 use magnus::{Error as RbErr, Ruby, TryConvert, Value, prelude::*};
 
 use crate::RbResult;
+use crate::utils::Wrap;
 
-pub struct RbArrowType<T>(pub T);
-
-impl TryConvert for RbArrowType<ArrowArrayStreamReader> {
+impl TryConvert for Wrap<ArrowArrayStreamReader> {
     fn try_convert(val: Value) -> RbResult<Self> {
         let ruby = Ruby::get_with(val);
         let addr: usize = val.funcall("to_i", ())?;
@@ -14,9 +15,23 @@ impl TryConvert for RbArrowType<ArrowArrayStreamReader> {
         let stream_ptr =
             Box::new(unsafe { std::ptr::replace(addr as _, FFI_ArrowArrayStream::empty()) });
 
-        Ok(RbArrowType(
-            ArrowArrayStreamReader::try_new(*stream_ptr)
-                .map_err(|e| RbErr::new(ruby.exception_arg_error(), e.to_string()))?,
-        ))
+        Ok(Wrap(ArrowArrayStreamReader::try_new(*stream_ptr).map_err(
+            |e| RbErr::new(ruby.exception_arg_error(), e.to_string()),
+        )?))
+    }
+}
+
+impl TryConvert for Wrap<ArrowSchema> {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        let ruby = Ruby::get_with(val);
+        let addr: usize = val.funcall("to_i", ())?;
+
+        // use similar approach as Polars to consume pointer and avoid copy
+        let schema_ptr =
+            Box::new(unsafe { std::ptr::replace(addr as _, FFI_ArrowSchema::empty()) });
+
+        Ok(Wrap(ArrowSchema::try_from(&*schema_ptr).map_err(|e| {
+            RbErr::new(ruby.exception_arg_error(), e.to_string())
+        })?))
     }
 }

data/ext/iceberg/src/batch.rs

@@ -0,0 +1,174 @@
+use std::sync::Arc;
+
+use arrow::array::{
+    Array, BooleanBuilder, Date32Builder, Decimal128Builder, Float32Builder, Float64Builder,
+    Int32Builder, Int64Builder, LargeBinaryBuilder, StringBuilder, TimestampMicrosecondBuilder,
+};
+use arrow::datatypes::{DataType as ArrowDataType, Decimal128Type};
+use arrow_array::ffi_stream::FFI_ArrowArrayStream;
+use arrow_array::{RecordBatch, RecordBatchIterator};
+use arrow_cast::parse::parse_decimal;
+use arrow_schema::{Field as ArrowField, Schema as ArrowSchema, TimeUnit};
+use magnus::{RArray, RHash, RString, Ruby, TryConvert, Value, value::ReprValue};
+
+use crate::RbResult;
+use crate::capsule::RbCapsule;
+use crate::error::todo_error;
+use crate::utils::{Wrap, date_to_i32};
+
+#[magnus::wrap(class = "Iceberg::ArrowRecordBatch")]
+pub struct RbArrowRecordBatch {
+    pub(crate) batch: RecordBatch,
+}
+
+impl RbArrowRecordBatch {
+    pub fn new(ruby: &Ruby, data: RArray, schema: Wrap<ArrowSchema>) -> RbResult<Self> {
+        let schema = Arc::new(schema.0);
+        let mut columns = Vec::new();
+        for field in &schema.fields {
+            let array = match field.data_type() {
+                ArrowDataType::Boolean => new_array_boolean(ruby, data, field)?,
+                ArrowDataType::Int32 => new_array_int32(ruby, data, field)?,
+                ArrowDataType::Int64 => new_array_int64(ruby, data, field)?,
+                ArrowDataType::Float32 => new_array_float32(ruby, data, field)?,
+                ArrowDataType::Float64 => new_array_float64(ruby, data, field)?,
+                ArrowDataType::Decimal128(precision, scale) => {
+                    new_array_decimal128(ruby, data, field, *precision, *scale)?
+                }
+                ArrowDataType::Date32 => new_array_date32(ruby, data, field)?,
+                ArrowDataType::Timestamp(TimeUnit::Microsecond, None) => {
+                    new_array_timestamp_us(ruby, data, field)?
+                }
+                ArrowDataType::Utf8 => new_array_utf8(ruby, data, field)?,
+                ArrowDataType::LargeBinary => new_array_large_binary(ruby, data, field)?,
+                _ => return Err(todo_error(field.data_type())),
+            };
+            columns.push(array);
+        }
+        let batch = RecordBatch::try_new(schema.clone(), columns).unwrap();
+        Ok(Self { batch })
+    }
+
+    pub fn arrow_c_stream(&self) -> RbCapsule {
+        let schema = self.batch.schema();
+        let reader = RecordBatchIterator::new([Ok(self.batch.clone())].into_iter(), schema);
+        let stream = FFI_ArrowArrayStream::new(Box::new(reader));
+        RbCapsule::new(stream, Some("arrow_array_stream".into()))
+    }
+}
+
+macro_rules! new_array {
+    ($name:ident, $builder:ty, $type:ty) => {
+        fn $name(ruby: &Ruby, data: RArray, field: &Arc<ArrowField>) -> RbResult<Arc<dyn Array>> {
+            let name = ruby.str_new(field.name());
+            let mut builder = <$builder>::new();
+            for row in data.into_iter() {
+                let row = RHash::try_convert(row)?;
+                let value: Option<$type> = row.aref(name)?;
+                match value {
+                    Some(v) => builder.append_value(v),
+                    None => builder.append_null(),
+                }
+            }
+            let array: Arc<dyn Array> = Arc::new(builder.finish());
+            Ok(array)
+        }
+    };
+}
+
+new_array!(new_array_boolean, BooleanBuilder, bool);
+new_array!(new_array_int32, Int32Builder, i32);
+new_array!(new_array_int64, Int64Builder, i64);
+new_array!(new_array_float32, Float32Builder, f32);
+new_array!(new_array_float64, Float64Builder, f64);
+new_array!(new_array_utf8, StringBuilder, String);
+
+fn new_array_decimal128(
+    ruby: &Ruby,
+    data: RArray,
+    field: &Arc<ArrowField>,
+    precision: u8,
+    scale: i8,
+) -> RbResult<Arc<dyn Array>> {
+    let name = ruby.str_new(field.name());
+    let mut builder = Decimal128Builder::new()
+        .with_precision_and_scale(precision, scale)
+        .unwrap();
+    for row in data.into_iter() {
+        let row = RHash::try_convert(row)?;
+        let value: Option<Value> = row.aref(name)?;
+        match value {
+            Some(v) => {
+                let s = v.to_r_string()?;
+                builder.append_value(
+                    parse_decimal::<Decimal128Type>(unsafe { s.as_str()? }, precision, scale)
+                        .unwrap(),
+                )
+            }
+            None => builder.append_null(),
+        }
+    }
+    let array: Arc<dyn Array> = Arc::new(builder.finish());
+    Ok(array)
+}
+
+fn new_array_date32(
+    ruby: &Ruby,
+    data: RArray,
+    field: &Arc<ArrowField>,
+) -> RbResult<Arc<dyn Array>> {
+    let name = ruby.str_new(field.name());
+    let mut builder = Date32Builder::new();
+    for row in data.into_iter() {
+        let row = RHash::try_convert(row)?;
+        let value: Option<Value> = row.aref(name)?;
+        match value {
+            Some(v) => builder.append_value(date_to_i32(v)?),
+            None => builder.append_null(),
+        }
+    }
+    let array: Arc<dyn Array> = Arc::new(builder.finish());
+    Ok(array)
+}
+
+fn new_array_timestamp_us(
+    ruby: &Ruby,
+    data: RArray,
+    field: &Arc<ArrowField>,
+) -> RbResult<Arc<dyn Array>> {
+    let name = ruby.str_new(field.name());
+    let mut builder = TimestampMicrosecondBuilder::new();
+    for row in data.into_iter() {
+        let row = RHash::try_convert(row)?;
+        let value: Option<Value> = row.aref(name)?;
+        match value {
+            Some(v) => {
+                let sec: i64 = v.funcall("to_i", ())?;
+                let usec: i64 = v.funcall("usec", ())?;
+                builder.append_value(sec * 1_000_000 + usec);
+            }
+            None => builder.append_null(),
+        }
+    }
+    let array: Arc<dyn Array> = Arc::new(builder.finish());
+    Ok(array)
+}
+
+fn new_array_large_binary(
+    ruby: &Ruby,
+    data: RArray,
+    field: &Arc<ArrowField>,
+) -> RbResult<Arc<dyn Array>> {
+    let name = ruby.str_new(field.name());
+    let mut builder = LargeBinaryBuilder::new();
+    for row in data.into_iter() {
+        let row = RHash::try_convert(row)?;
+        let value: Option<RString> = row.aref(name)?;
+        match value {
+            Some(v) => builder.append_value(unsafe { v.as_slice() }),
+            None => builder.append_null(),
+        }
+    }
+    let array: Arc<dyn Array> = Arc::new(builder.finish());
+    Ok(array)
+}

data/ext/iceberg/src/capsule.rs

@@ -0,0 +1,24 @@
+use std::any::Any;
+
+#[magnus::wrap(class = "Iceberg::Capsule")]
+pub struct RbCapsule {
+    value: Box<dyn Any + Send>,
+    name: Option<String>,
+}
+
+impl RbCapsule {
+    pub fn new<T: 'static + Send>(value: T, name: Option<String>) -> Self {
+        Self {
+            value: Box::new(value),
+            name,
+        }
+    }
+
+    pub fn to_i(&self) -> usize {
+        (&*self.value as *const dyn Any as *const ()) as usize
+    }
+
+    pub fn name(&self) -> Option<&str> {
+        self.name.as_deref()
+    }
+}