iceberg 0.12.0 → 0.12.1

data/README.md

@@ -31,10 +31,17 @@ catalog.create_namespace("main")
 Create a table
 
 ```ruby
-catalog.create_table("events") do |t|
-  t.bigint "id"
-  t.float "value"
-end
+table =
+  catalog.create_table("events") do |t|
+    t.bigint "id"
+    t.float "value"
+  end
+```
+
+Append data
+
+```ruby
+table.append([{"id" => 1, "value" => 3.0}, {"id" => 2, "value" => 4.0}])
 ```
 
 Or with [Polars](https://github.com/ankane/ruby-polars)
@@ -45,6 +52,18 @@ table = catalog.create_table("events", schema: df.schema)
 table.append(df)
 ```
 
+Or with [Nanoarrow](https://github.com/ankane/nanoarrow-ruby)
+
+```ruby
+arr =
+  Nanoarrow::Array.new(
+    [{"id" => 1, "value" => 3.0}, {"id" => 2, "value" => 4.0}],
+    Nanoarrow.struct({"a" => Nanoarrow.int64, "b" => Nanoarrow.string})
+  )
+table = catalog.create_table("events", schema: arr.schema)
+table.append(arr)
+```
+
 Load a table
 
 ```ruby
@@ -54,7 +73,11 @@ table = catalog.load_table("events")
 Query a table
 
 ```ruby
+table.to_a
+# or
 table.to_polars.collect
+# or
+table.scan.to_arrow
 ```
 
 ## Catalog Types
@@ -69,8 +92,6 @@ Iceberg::RestCatalog.new(
 
 ### S3 Tables
 
-[unreleased]
-
 ```ruby
 Iceberg::S3TablesCatalog.new(
   arn: "arn:aws:s3tables:..."
@@ -79,8 +100,6 @@ Iceberg::S3TablesCatalog.new(
 
 ### Glue
 
-[unreleased]
-
 ```ruby
 Iceberg::GlueCatalog.new(
   warehouse: "s3://my-bucket"
@@ -228,4 +247,10 @@ bundle exec rake test:rest
 # SQL catalog
 createdb iceberg_ruby_test
 bundle exec rake test:sql
+
+# S3 Tables catalog
+bundle exec rake test:s3tables
+
+# Glue catalog
+bundle exec rake test:glue
 ```

data/ext/iceberg/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "iceberg-ruby"
-version = "0.12.0"
+version = "0.12.1"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2024"
-rust-version = "1.94"
+rust-version = "1.95"
 publish = false
 
 [lib]
@@ -12,23 +12,23 @@ name = "iceberg"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = "58"
-arrow-array = { version = "58", features = ["ffi"] }
-arrow-cast = "58"
-arrow-schema = "58"
-datafusion = { version = "53", optional = true }
+arrow = "58.4"
+arrow-array = { version = "58.4", features = ["ffi"] }
+arrow-cast = "58.4"
+arrow-schema = "58.4"
+datafusion = { version = "54", optional = true }
 futures = "0.3"
-iceberg = "=0.10.0"
-iceberg-catalog-glue = { version = "=0.10.0", optional = true }
-iceberg-catalog-rest = { version = "=0.10.0", optional = true }
-iceberg-catalog-s3tables = { version = "=0.10.0", optional = true }
-iceberg-catalog-sql = { version = "=0.10.0", optional = true }
-iceberg-datafusion = { version = "=0.10.0", optional = true }
-iceberg-storage-opendal = "=0.10.0"
-magnus = "0.8"
-parquet = "58"
+iceberg = "=0.11.0"
+iceberg-catalog-glue = { version = "=0.11.0", optional = true }
+iceberg-catalog-rest = { version = "=0.11.0", optional = true }
+iceberg-catalog-s3tables = { version = "=0.11.0", optional = true }
+iceberg-catalog-sql = { version = "=0.11.0", optional = true }
+iceberg-datafusion = { version = "=0.11.0", optional = true }
+iceberg-storage-opendal = "=0.11.0"
+magnus = "0.9"
+parquet = "58.4"
 rb-sys = "0.9"
-sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite"], default-features = false, optional = true }
+sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite", "tls-rustls-aws-lc-rs"], default-features = false, optional = true }
 tokio = { version = "1", features = ["rt-multi-thread"] }
 uuid = { version = "1", features = ["v4"] }
 

data/ext/iceberg/src/catalog.rs

@@ -316,21 +316,18 @@ impl RbCatalog {
         partition_spec: Option<&RbPartitionSpec>,
         sort_order: Option<&RbSortOrder>,
         properties: HashMap<String, String>,
+        format_version: u8,
     ) -> RbResult<RbTable> {
-        let format_version = if let Some(v) = properties.get("format-version") {
-            match v.as_str() {
-                "1" => FormatVersion::V1,
-                "2" => FormatVersion::V2,
-                "3" => FormatVersion::V3,
-                _ => {
-                    return Err(RbErr::new(
-                        ruby.exception_arg_error(),
-                        "Unsupported format version",
-                    ));
-                }
+        let format_version = match format_version {
+            1 => FormatVersion::V1,
+            2 => FormatVersion::V2,
+            3 => FormatVersion::V3,
+            _ => {
+                return Err(RbErr::new(
+                    ruby.exception_arg_error(),
+                    "Unsupported format version",
+                ));
             }
-        } else {
-            FormatVersion::V2
         };
         let creation = TableCreation::builder()
             .name(name.0.name)

data/ext/iceberg/src/error.rs

@@ -1,3 +1,5 @@
+use std::error::Error as _;
+
 use iceberg::{Error, ErrorKind};
 use magnus::{Error as RbErr, RModule, Ruby, prelude::*};
 
@@ -14,7 +16,18 @@ pub fn to_rb_err(err: Error) -> RbErr {
 
     // no way to get context separately
     // https://github.com/apache/iceberg-rust/issues/1071
-    let message = err.message().to_string();
+    let mut message = err.message().to_string();
+
+    let mut source = err.source();
+    while let Some(s) = source {
+        let m = s.to_string();
+        if message.contains(&m) {
+            break;
+        }
+        message.push_str(": ");
+        message.push_str(&m);
+        source = s.source();
+    }
 
     if class_name == "Error" {
         let s = err.to_string();

data/ext/iceberg/src/lib.rs

@@ -58,7 +58,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("update_namespace", method!(RbCatalog::update_namespace, 2))?;
     class.define_method("drop_namespace", method!(RbCatalog::drop_namespace, 1))?;
     class.define_method("list_tables", method!(RbCatalog::list_tables, 1))?;
-    class.define_method("create_table", method!(RbCatalog::create_table, 6))?;
+    class.define_method("create_table", method!(RbCatalog::create_table, 7))?;
     class.define_method("load_table", method!(RbCatalog::load_table, 1))?;
     class.define_method("drop_table", method!(RbCatalog::drop_table, 1))?;
     class.define_method("purge_table", method!(RbCatalog::purge_table, 1))?;

data/ext/iceberg/src/scan.rs

@@ -82,15 +82,15 @@ impl RbTableScan {
 impl RbFileScanTask {
     pub fn file(&self) -> RbDataFile {
         RbDataFile {
-            file_path: self.scan.data_file_path.clone(),
-            record_count: self.scan.record_count,
-            file_size_in_bytes: self.scan.file_size_in_bytes,
+            file_path: self.scan.data_file_path().to_string(),
+            record_count: self.scan.record_count(),
+            file_size_in_bytes: self.scan.file_size_in_bytes(),
             equality_ids: None,
         }
     }
 
     pub fn delete_files(ruby: &Ruby, rb_self: &Self) -> RArray {
-        ruby.ary_from_iter(rb_self.scan.deletes.iter().map(|v| RbDataFile {
+        ruby.ary_from_iter(rb_self.scan.deletes().iter().map(|v| RbDataFile {
             file_path: v.file_path.clone(),
             record_count: None,
             file_size_in_bytes: v.file_size_in_bytes,

data/ext/iceberg/src/schema.rs

@@ -10,7 +10,7 @@ use magnus::{
 
 use crate::RbResult;
 use crate::capsule::RbCapsule;
-use crate::error::to_rb_err;
+use crate::error::{to_rb_err, todo_error};
 use crate::utils::{Wrap, default_value, rb_literal};
 
 #[magnus::wrap(class = "Iceberg::Schema")]
@@ -252,6 +252,7 @@ impl RbNestedField {
                 RbNestedField::from(&ty.key_field),
                 RbNestedField::from(&ty.value_field),
             ))?,
+            Type::Variant(_) => return Err(todo_error(field_type)),
         };
         Ok(v)
     }

data/ext/iceberg/src/utils.rs

@@ -160,6 +160,7 @@ pub fn default_value(ob: Value, field_type: &Type) -> RbResult<Option<Literal>>
         Type::Struct(_) => return Err(todo_error(field_type)),
         Type::List(_) => return Err(todo_error(field_type)),
         Type::Map(_) => return Err(todo_error(field_type)),
+        Type::Variant(_) => return Err(todo_error(field_type)),
     };
     Ok(Some(lit))
 }

data/lib/iceberg/catalog.rb

@@ -66,7 +66,12 @@ module Iceberg
         schema = Schema.new
       end
 
-      Table.new(@catalog.create_table(with_namespace(table_name), schema, location, partition_spec, sort_order, properties), @catalog)
+      format_version = (properties["format-version"] || 2).to_i
+      if properties["format-version"] && (is_a?(MemoryCatalog) || is_a?(SqlCatalog))
+        properties = properties.except("format-version")
+      end
+
+      Table.new(@catalog.create_table(with_namespace(table_name), schema, location, partition_spec, sort_order, properties, format_version), @catalog)
     end
 
     def load_table(table_name)

data/lib/iceberg/version.rb

@@ -1,3 +1,3 @@
 module Iceberg
-  VERSION = "0.12.0"
+  VERSION = "0.12.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iceberg
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.12.1
 platform: ruby
 authors:
 - Andrew Kane
@@ -88,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.14
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Apache Iceberg for Ruby
 test_files: []