iceberg 0.10.0 → 0.10.1

data/Cargo.toml

@@ -0,0 +1,6 @@
+[workspace]
+members = ["ext/iceberg"]
+resolver = "3"
+
+[profile.release]
+strip = true

data/README.md

@@ -12,6 +12,8 @@ Add this line to your application’s Gemfile:
 gem "iceberg"
 ```
 
+It can take around 5 minutes to compile the gem.
+
 ## Getting Started
 
 Create a client for an Iceberg catalog

data/ext/iceberg/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "iceberg"
+version = "0.10.1"
+license = "Apache-2.0"
+authors = ["Andrew Kane <andrew@ankane.org>"]
+edition = "2024"
+rust-version = "1.86"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+arrow-array = { version = "55", features = ["ffi"] }
+arrow-schema = "55"
+datafusion = { version = "47", optional = true }
+futures = "0.3"
+iceberg = "0.6"
+iceberg-catalog-glue = { version = "0.6", optional = true }
+iceberg-catalog-rest = { version = "0.6", optional = true }
+iceberg-catalog-sql = { version = "0.6", optional = true }
+iceberg-datafusion = { version = "0.6", optional = true }
+magnus = "0.8"
+parquet = "55"
+sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite"], default-features = false, optional = true }
+tokio = { version = "1", features = ["rt-multi-thread"] }
+uuid = { version = "1", features = ["v4"] }
+
+[features]
+default = ["rest", "sql"]
+datafusion = ["dep:datafusion", "dep:iceberg-datafusion"]
+glue = ["dep:iceberg-catalog-glue"]
+rest = ["dep:iceberg-catalog-rest"]
+sql = ["dep:iceberg-catalog-sql", "dep:sqlx"]

data/ext/iceberg/src/arrow.rs

@@ -0,0 +1,22 @@
+use arrow_array::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
+use magnus::{Error as RbErr, Ruby, TryConvert, Value, prelude::*};
+
+use crate::RbResult;
+
+pub struct RbArrowType<T>(pub T);
+
+impl TryConvert for RbArrowType<ArrowArrayStreamReader> {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        let ruby = Ruby::get_with(val);
+        let addr: usize = val.funcall("to_i", ())?;
+
+        // use similar approach as Polars to consume pointer and avoid copy
+        let stream_ptr =
+            Box::new(unsafe { std::ptr::replace(addr as _, FFI_ArrowArrayStream::empty()) });
+
+        Ok(RbArrowType(
+            ArrowArrayStreamReader::try_new(*stream_ptr)
+                .map_err(|e| RbErr::new(ruby.exception_arg_error(), e.to_string()))?,
+        ))
+    }
+}

data/ext/iceberg/src/catalog.rs

@@ -0,0 +1,315 @@
+#[cfg(feature = "datafusion")]
+use datafusion::execution::context::SessionContext;
+use iceberg::io::{FileIO, FileIOBuilder};
+use iceberg::spec::Schema;
+use iceberg::{Catalog, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
+#[cfg(feature = "glue")]
+use iceberg_catalog_glue::{GlueCatalog, GlueCatalogConfig};
+#[cfg(feature = "rest")]
+use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig};
+#[cfg(feature = "sql")]
+use iceberg_catalog_sql::{SqlBindStyle, SqlCatalog, SqlCatalogConfig};
+#[cfg(feature = "datafusion")]
+use iceberg_datafusion::IcebergCatalogProvider;
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::error::to_rb_err;
+use crate::runtime::runtime;
+use crate::utils::Wrap;
+use crate::{RbResult, RbTable};
+
+pub enum RbCatalogType {
+    #[cfg(feature = "glue")]
+    Glue(Arc<GlueCatalog>),
+    Memory(Arc<MemoryCatalog>),
+    #[cfg(feature = "rest")]
+    Rest(Arc<RestCatalog>),
+    #[cfg(feature = "sql")]
+    Sql(Arc<SqlCatalog>),
+}
+
+impl RbCatalogType {
+    pub fn as_catalog(&self) -> &dyn Catalog {
+        match self {
+            #[cfg(feature = "glue")]
+            RbCatalogType::Glue(v) => v.as_ref(),
+            RbCatalogType::Memory(v) => v.as_ref(),
+            #[cfg(feature = "rest")]
+            RbCatalogType::Rest(v) => v.as_ref(),
+            #[cfg(feature = "sql")]
+            RbCatalogType::Sql(v) => v.as_ref(),
+        }
+    }
+
+    #[cfg(feature = "datafusion")]
+    fn as_arc(&self) -> Arc<dyn Catalog> {
+        match self {
+            #[cfg(feature = "glue")]
+            RbCatalogType::Glue(v) => v.clone(),
+            RbCatalogType::Memory(v) => v.clone(),
+            #[cfg(feature = "rest")]
+            RbCatalogType::Rest(v) => v.clone(),
+            #[cfg(feature = "sql")]
+            RbCatalogType::Sql(v) => v.clone(),
+        }
+    }
+}
+
+#[magnus::wrap(class = "Iceberg::RbCatalog")]
+pub struct RbCatalog {
+    pub catalog: RefCell<RbCatalogType>,
+}
+
+impl RbCatalog {
+    #[cfg(feature = "glue")]
+    pub fn new_glue(warehouse: String) -> RbResult<Self> {
+        let config = GlueCatalogConfig::builder().warehouse(warehouse).build();
+        let catalog = runtime()
+            .block_on(GlueCatalog::new(config))
+            .map_err(to_rb_err)?;
+        Ok(Self {
+            catalog: RbCatalogType::Glue(catalog.into()).into(),
+        })
+    }
+
+    pub fn new_memory(warehouse: Option<String>) -> RbResult<Self> {
+        let file_io = match warehouse {
+            Some(ref v) => FileIO::from_path(v)
+                .map_err(to_rb_err)?
+                .build()
+                .map_err(to_rb_err)?,
+            None => FileIOBuilder::new_fs_io().build().map_err(to_rb_err)?,
+        };
+        let catalog = MemoryCatalog::new(file_io, warehouse);
+        Ok(Self {
+            catalog: RbCatalogType::Memory(catalog.into()).into(),
+        })
+    }
+
+    #[cfg(feature = "rest")]
+    pub fn new_rest(
+        uri: String,
+        warehouse: Option<String>,
+        props: HashMap<String, String>,
+    ) -> Self {
+        let config = RestCatalogConfig::builder()
+            .uri(uri)
+            .warehouse_opt(warehouse)
+            .props(props)
+            .build();
+        let catalog = RestCatalog::new(config);
+        Self {
+            catalog: RbCatalogType::Rest(catalog.into()).into(),
+        }
+    }
+
+    #[cfg(feature = "sql")]
+    pub fn new_sql(
+        uri: String,
+        warehouse: String,
+        name: String,
+        props: HashMap<String, String>,
+    ) -> RbResult<Self> {
+        let file_io = FileIO::from_path(&warehouse)
+            .map_err(to_rb_err)?
+            .build()
+            .map_err(to_rb_err)?;
+        let config = SqlCatalogConfig::builder()
+            .uri(uri)
+            .warehouse_location(warehouse)
+            .name(name)
+            .file_io(file_io)
+            .sql_bind_style(SqlBindStyle::DollarNumeric)
+            .props(props)
+            .build();
+        let catalog = runtime()
+            .block_on(SqlCatalog::new(config))
+            .map_err(to_rb_err)?;
+        Ok(Self {
+            catalog: RbCatalogType::Sql(catalog.into()).into(),
+        })
+    }
+
+    pub fn list_namespaces(
+        &self,
+        parent: Option<Wrap<NamespaceIdent>>,
+    ) -> RbResult<Vec<Vec<String>>> {
+        let namespaces = runtime()
+            .block_on(
+                self.catalog
+                    .borrow()
+                    .as_catalog()
+                    .list_namespaces(parent.map(|v| v.0).as_ref()),
+            )
+            .map_err(to_rb_err)?;
+        Ok(namespaces.iter().map(|v| v.clone().inner()).collect())
+    }
+
+    pub fn create_namespace(
+        &self,
+        name: Wrap<NamespaceIdent>,
+        props: HashMap<String, String>,
+    ) -> RbResult<()> {
+        runtime()
+            .block_on(
+                self.catalog
+                    .borrow()
+                    .as_catalog()
+                    .create_namespace(&name.0, props),
+            )
+            .map_err(to_rb_err)?;
+        Ok(())
+    }
+
+    pub fn namespace_exists(&self, name: Wrap<NamespaceIdent>) -> RbResult<bool> {
+        let exists = runtime()
+            .block_on(self.catalog.borrow().as_catalog().namespace_exists(&name.0))
+            .map_err(to_rb_err)?;
+        Ok(exists)
+    }
+
+    pub fn namespace_properties(
+        &self,
+        name: Wrap<NamespaceIdent>,
+    ) -> RbResult<HashMap<String, String>> {
+        let namespace = runtime()
+            .block_on(self.catalog.borrow().as_catalog().get_namespace(&name.0))
+            .map_err(to_rb_err)?;
+        Ok(namespace.properties().clone())
+    }
+
+    pub fn update_namespace(
+        &self,
+        name: Wrap<NamespaceIdent>,
+        props: HashMap<String, String>,
+    ) -> RbResult<()> {
+        runtime()
+            .block_on(
+                self.catalog
+                    .borrow()
+                    .as_catalog()
+                    .update_namespace(&name.0, props),
+            )
+            .map_err(to_rb_err)?;
+        Ok(())
+    }
+
+    pub fn drop_namespace(&self, name: Wrap<NamespaceIdent>) -> RbResult<()> {
+        runtime()
+            .block_on(self.catalog.borrow().as_catalog().drop_namespace(&name.0))
+            .map_err(to_rb_err)?;
+        Ok(())
+    }
+
+    pub fn list_tables(&self, namespace: Wrap<NamespaceIdent>) -> RbResult<Vec<Vec<String>>> {
+        let tables = runtime()
+            .block_on(self.catalog.borrow().as_catalog().list_tables(&namespace.0))
+            .map_err(to_rb_err)?;
+        Ok(tables
+            .iter()
+            .map(|v| {
+                let mut vec = v.namespace.clone().inner();
+                vec.push(v.name.clone());
+                vec
+            })
+            .collect())
+    }
+
+    pub fn create_table(
+        &self,
+        name: Wrap<TableIdent>,
+        schema: Wrap<Schema>,
+        location: Option<String>,
+    ) -> RbResult<RbTable> {
+        let creation = TableCreation::builder()
+            .name(name.0.name)
+            .schema(schema.0)
+            .location_opt(location)
+            .build();
+        let table = runtime()
+            .block_on(
+                self.catalog
+                    .borrow()
+                    .as_catalog()
+                    .create_table(&name.0.namespace, creation),
+            )
+            .map_err(to_rb_err)?;
+        Ok(RbTable {
+            table: table.into(),
+        })
+    }
+
+    pub fn load_table(&self, name: Wrap<TableIdent>) -> RbResult<RbTable> {
+        let table = runtime()
+            .block_on(self.catalog.borrow().as_catalog().load_table(&name.0))
+            .map_err(to_rb_err)?;
+        Ok(RbTable {
+            table: table.into(),
+        })
+    }
+
+    pub fn drop_table(&self, name: Wrap<TableIdent>) -> RbResult<()> {
+        runtime()
+            .block_on(self.catalog.borrow().as_catalog().drop_table(&name.0))
+            .map_err(to_rb_err)?;
+        Ok(())
+    }
+
+    pub fn table_exists(&self, name: Wrap<TableIdent>) -> RbResult<bool> {
+        let exists = runtime()
+            .block_on(self.catalog.borrow().as_catalog().table_exists(&name.0))
+            .map_err(to_rb_err)?;
+        Ok(exists)
+    }
+
+    pub fn rename_table(&self, name: Wrap<TableIdent>, new_name: Wrap<TableIdent>) -> RbResult<()> {
+        runtime()
+            .block_on(
+                self.catalog
+                    .borrow()
+                    .as_catalog()
+                    .rename_table(&name.0, &new_name.0),
+            )
+            .map_err(to_rb_err)?;
+        Ok(())
+    }
+
+    pub fn register_table(
+        &self,
+        name: Wrap<TableIdent>,
+        metadata_location: String,
+    ) -> RbResult<()> {
+        runtime()
+            .block_on(
+                self.catalog
+                    .borrow()
+                    .as_catalog()
+                    .register_table(&name.0, metadata_location),
+            )
+            .map_err(to_rb_err)?;
+        Ok(())
+    }
+
+    #[cfg(feature = "datafusion")]
+    pub fn query(&self, sql: String) -> RbResult<()> {
+        let runtime = runtime();
+
+        // TODO only create context once
+        let catalog = self.catalog.borrow().as_arc();
+        let provider = runtime
+            .block_on(IcebergCatalogProvider::try_new(catalog))
+            .unwrap();
+        let ctx = SessionContext::new();
+        ctx.register_catalog("datafusion", Arc::new(provider));
+
+        let df = runtime.block_on(ctx.sql(&sql)).unwrap();
+        let _results = runtime.block_on(df.collect()).unwrap();
+
+        // println!("{:?}", df.schema().fields());
+        // println!("{:?}", results);
+
+        Ok(())
+    }
+}

data/ext/iceberg/src/error.rs

@@ -0,0 +1,32 @@
+use magnus::{Error as RbErr, RModule, Ruby, prelude::*};
+
+pub fn to_rb_err(err: iceberg::Error) -> RbErr {
+    let class_name = match err.kind() {
+        iceberg::ErrorKind::NamespaceAlreadyExists => "NamespaceAlreadyExistsError",
+        iceberg::ErrorKind::NamespaceNotFound => "NamespaceNotFoundError",
+        iceberg::ErrorKind::TableAlreadyExists => "TableAlreadyExistsError",
+        iceberg::ErrorKind::TableNotFound => "TableNotFoundError",
+        iceberg::ErrorKind::FeatureUnsupported => "UnsupportedFeatureError",
+        iceberg::ErrorKind::DataInvalid => "InvalidDataError",
+        _ => "Error",
+    };
+
+    let class = Ruby::get()
+        .unwrap()
+        .class_object()
+        .const_get::<_, RModule>("Iceberg")
+        .unwrap()
+        .const_get(class_name)
+        .unwrap();
+
+    // no way to get context separately
+    // https://github.com/apache/iceberg-rust/issues/1071
+    let message = err.to_string();
+    let message = message
+        // TODO improve
+        .strip_prefix("Unexpected => ")
+        .map(|v| v.to_string())
+        .unwrap_or(message);
+
+    RbErr::new(class, message)
+}

data/ext/iceberg/src/lib.rs

@@ -0,0 +1,127 @@
+mod arrow;
+mod catalog;
+mod error;
+mod runtime;
+mod scan;
+mod table;
+mod utils;
+
+use magnus::{Error as RbErr, Ruby, function, method, prelude::*};
+
+use crate::catalog::RbCatalog;
+use crate::scan::RbTableScan;
+use crate::table::RbTable;
+
+type RbResult<T> = Result<T, RbErr>;
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> RbResult<()> {
+    let module = ruby.define_module("Iceberg")?;
+
+    let class = module.define_class("RbCatalog", ruby.class_object())?;
+    #[cfg(feature = "glue")]
+    class.define_singleton_method("new_glue", function!(RbCatalog::new_glue, 1))?;
+    class.define_singleton_method("new_memory", function!(RbCatalog::new_memory, 1))?;
+    #[cfg(feature = "rest")]
+    class.define_singleton_method("new_rest", function!(RbCatalog::new_rest, 3))?;
+    #[cfg(feature = "sql")]
+    class.define_singleton_method("new_sql", function!(RbCatalog::new_sql, 4))?;
+    class.define_method("list_namespaces", method!(RbCatalog::list_namespaces, 1))?;
+    class.define_method("create_namespace", method!(RbCatalog::create_namespace, 2))?;
+    class.define_method("namespace_exists?", method!(RbCatalog::namespace_exists, 1))?;
+    class.define_method(
+        "namespace_properties",
+        method!(RbCatalog::namespace_properties, 1),
+    )?;
+    class.define_method("update_namespace", method!(RbCatalog::update_namespace, 2))?;
+    class.define_method("drop_namespace", method!(RbCatalog::drop_namespace, 1))?;
+    class.define_method("list_tables", method!(RbCatalog::list_tables, 1))?;
+    class.define_method("create_table", method!(RbCatalog::create_table, 3))?;
+    class.define_method("load_table", method!(RbCatalog::load_table, 1))?;
+    class.define_method("drop_table", method!(RbCatalog::drop_table, 1))?;
+    class.define_method("table_exists?", method!(RbCatalog::table_exists, 1))?;
+    class.define_method("rename_table", method!(RbCatalog::rename_table, 2))?;
+    class.define_method("register_table", method!(RbCatalog::register_table, 2))?;
+    #[cfg(feature = "datafusion")]
+    class.define_method("query", method!(RbCatalog::query, 1))?;
+
+    let class = module.define_class("RbTable", ruby.class_object())?;
+    class.define_method("scan", method!(RbTable::scan, 1))?;
+    class.define_method("append", method!(RbTable::append, 2))?;
+    class.define_method("format_version", method!(RbTable::format_version, 0))?;
+    class.define_method("uuid", method!(RbTable::uuid, 0))?;
+    class.define_method("location", method!(RbTable::location, 0))?;
+    class.define_method(
+        "last_sequence_number",
+        method!(RbTable::last_sequence_number, 0),
+    )?;
+    class.define_method(
+        "next_sequence_number",
+        method!(RbTable::next_sequence_number, 0),
+    )?;
+    class.define_method("last_column_id", method!(RbTable::last_column_id, 0))?;
+    class.define_method("last_partition_id", method!(RbTable::last_partition_id, 0))?;
+    class.define_method("last_updated_ms", method!(RbTable::last_updated_ms, 0))?;
+    class.define_method("schemas", method!(RbTable::schemas, 0))?;
+    class.define_method("schema_by_id", method!(RbTable::schema_by_id, 1))?;
+    class.define_method("current_schema", method!(RbTable::current_schema, 0))?;
+    class.define_method("current_schema_id", method!(RbTable::current_schema_id, 0))?;
+    class.define_method("partition_specs", method!(RbTable::partition_specs, 0))?;
+    class.define_method(
+        "partition_spec_by_id",
+        method!(RbTable::partition_spec_by_id, 1),
+    )?;
+    class.define_method(
+        "default_partition_spec",
+        method!(RbTable::default_partition_spec, 0),
+    )?;
+    class.define_method(
+        "default_partition_spec_id",
+        method!(RbTable::default_partition_spec_id, 0),
+    )?;
+    class.define_method("snapshots", method!(RbTable::snapshots, 0))?;
+    class.define_method("snapshot_by_id", method!(RbTable::snapshot_by_id, 1))?;
+    class.define_method("history", method!(RbTable::history, 0))?;
+    class.define_method("metadata_log", method!(RbTable::metadata_log, 0))?;
+    class.define_method("current_snapshot", method!(RbTable::current_snapshot, 0))?;
+    class.define_method(
+        "current_snapshot_id",
+        method!(RbTable::current_snapshot_id, 0),
+    )?;
+    class.define_method("snapshot_for_ref", method!(RbTable::snapshot_for_ref, 1))?;
+    class.define_method("sort_orders", method!(RbTable::sort_orders, 0))?;
+    class.define_method("sort_order_by_id", method!(RbTable::sort_order_by_id, 1))?;
+    class.define_method(
+        "default_sort_order",
+        method!(RbTable::default_sort_order, 0),
+    )?;
+    class.define_method(
+        "default_sort_order_id",
+        method!(RbTable::default_sort_order_id, 0),
+    )?;
+    class.define_method("properties", method!(RbTable::properties, 0))?;
+    class.define_method("statistics", method!(RbTable::statistics, 0))?;
+    class.define_method(
+        "partition_statistics",
+        method!(RbTable::partition_statistics, 0),
+    )?;
+    class.define_method(
+        "statistics_for_snapshot",
+        method!(RbTable::statistics_for_snapshot, 1),
+    )?;
+    class.define_method(
+        "partition_statistics_for_snapshot",
+        method!(RbTable::partition_statistics_for_snapshot, 1),
+    )?;
+    class.define_method("encryption_keys", method!(RbTable::encryption_keys, 0))?;
+    class.define_method("encryption_key", method!(RbTable::encryption_key, 1))?;
+    class.define_singleton_method(
+        "from_metadata_file",
+        function!(RbTable::from_metadata_file, 1),
+    )?;
+
+    let class = module.define_class("RbTableScan", ruby.class_object())?;
+    class.define_method("plan_files", method!(RbTableScan::plan_files, 0))?;
+
+    Ok(())
+}

data/ext/iceberg/src/runtime.rs

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::sync::OnceLock;
+
+use tokio::runtime::{Handle, Runtime};
+
+static RUNTIME: OnceLock<Runtime> = OnceLock::new();
+
+// TODO https://github.com/apache/iceberg-rust/pull/1396
+pub fn runtime() -> Handle {
+    match Handle::try_current() {
+        Ok(h) => h.clone(),
+        _ => {
+            let rt = RUNTIME.get_or_init(|| Runtime::new().unwrap());
+            rt.handle().clone()
+        }
+    }
+}

data/ext/iceberg/src/scan.rs

@@ -0,0 +1,47 @@
+use futures::TryStreamExt;
+use iceberg::scan::TableScan;
+use magnus::{RArray, Ruby};
+use std::cell::RefCell;
+
+use crate::RbResult;
+use crate::error::to_rb_err;
+use crate::runtime::runtime;
+
+#[magnus::wrap(class = "Iceberg::RbTableScan")]
+pub struct RbTableScan {
+    pub scan: RefCell<TableScan>,
+}
+
+impl RbTableScan {
+    pub fn plan_files(ruby: &Ruby, rb_self: &Self) -> RbResult<RArray> {
+        let scan = rb_self.scan.borrow();
+
+        let runtime = runtime();
+        let plan_files = runtime.block_on(scan.plan_files()).map_err(to_rb_err)?;
+        let plan_files: Vec<_> = runtime
+            .block_on(plan_files.try_collect())
+            .map_err(to_rb_err)?;
+        let files = ruby.ary_new();
+        for v in plan_files {
+            let file = ruby.hash_new();
+            file.aset(ruby.to_symbol("start"), v.start)?;
+            file.aset(ruby.to_symbol("length"), v.length)?;
+            file.aset(ruby.to_symbol("record_count"), v.record_count)?;
+            file.aset(ruby.to_symbol("data_file_path"), v.data_file_path)?;
+            file.aset(ruby.to_symbol("project_field_ids"), v.project_field_ids)?;
+
+            let deletes = ruby.ary_new();
+            for d in v.deletes {
+                let delete = ruby.hash_new();
+                delete.aset(ruby.to_symbol("file_path"), d.file_path)?;
+                delete.aset(ruby.to_symbol("partition_spec_id"), d.partition_spec_id)?;
+                delete.aset(ruby.to_symbol("equality_ids"), d.equality_ids)?;
+                deletes.push(delete)?;
+            }
+            file.aset(ruby.to_symbol("deletes"), deletes)?;
+
+            files.push(file)?;
+        }
+        Ok(files)
+    }
+}