iceberg 0.10.1 → 0.10.3

data/README.md

@@ -12,8 +12,6 @@ Add this line to your application’s Gemfile:
 gem "iceberg"
 ```
 
-It can take around 5 minutes to compile the gem.
-
 ## Getting Started
 
 Create a client for an Iceberg catalog
@@ -84,9 +82,7 @@ Iceberg::MemoryCatalog.new(
 )
 ```
 
-## Reference
-
-### Namespaces
+## Namespaces
 
 List namespaces
 
@@ -124,7 +120,7 @@ Drop a namespace
 catalog.drop_namespace("main")
 ```
 
-### Tables
+## Tables
 
 List tables
 
@@ -171,7 +167,7 @@ Drop a table
 catalog.drop_table("main.events")
 ```
 
-### Static Tables
+## Static Tables
 
 Load a static table
 

data/ext/iceberg/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "iceberg"
-version = "0.10.1"
+version = "0.10.3"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2024"
-rust-version = "1.86"
+rust-version = "1.87"
 publish = false
 
 [lib]
@@ -13,13 +13,13 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow-array = { version = "55", features = ["ffi"] }
 arrow-schema = "55"
-datafusion = { version = "47", optional = true }
+datafusion = { version = "48", optional = true }
 futures = "0.3"
-iceberg = "0.6"
-iceberg-catalog-glue = { version = "0.6", optional = true }
-iceberg-catalog-rest = { version = "0.6", optional = true }
-iceberg-catalog-sql = { version = "0.6", optional = true }
-iceberg-datafusion = { version = "0.6", optional = true }
+iceberg = "0.7"
+iceberg-catalog-glue = { version = "0.7", optional = true }
+iceberg-catalog-rest = { version = "0.7", optional = true }
+iceberg-catalog-sql = { version = "0.7", optional = true }
+iceberg-datafusion = { version = "0.7", optional = true }
 magnus = "0.8"
 parquet = "55"
 sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite"], default-features = false, optional = true }

data/ext/iceberg/src/catalog.rs

@@ -1,12 +1,15 @@
 #[cfg(feature = "datafusion")]
 use datafusion::execution::context::SessionContext;
-use iceberg::io::{FileIO, FileIOBuilder};
+use iceberg::io::FileIO;
+use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder};
 use iceberg::spec::Schema;
-use iceberg::{Catalog, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
+use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
 #[cfg(feature = "glue")]
-use iceberg_catalog_glue::{GlueCatalog, GlueCatalogConfig};
+use iceberg_catalog_glue::{GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder};
 #[cfg(feature = "rest")]
-use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig};
+use iceberg_catalog_rest::{
+    REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RestCatalog, RestCatalogBuilder,
+};
 #[cfg(feature = "sql")]
 use iceberg_catalog_sql::{SqlBindStyle, SqlCatalog, SqlCatalogConfig};
 #[cfg(feature = "datafusion")]
@@ -65,9 +68,9 @@ pub struct RbCatalog {
 impl RbCatalog {
     #[cfg(feature = "glue")]
     pub fn new_glue(warehouse: String) -> RbResult<Self> {
-        let config = GlueCatalogConfig::builder().warehouse(warehouse).build();
+        let props = HashMap::from([(GLUE_CATALOG_PROP_WAREHOUSE.to_string(), warehouse)]);
         let catalog = runtime()
-            .block_on(GlueCatalog::new(config))
+            .block_on(GlueCatalogBuilder::default().load("glue", props))
             .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Glue(catalog.into()).into(),
@@ -75,14 +78,13 @@ impl RbCatalog {
     }
 
     pub fn new_memory(warehouse: Option<String>) -> RbResult<Self> {
-        let file_io = match warehouse {
-            Some(ref v) => FileIO::from_path(v)
-                .map_err(to_rb_err)?
-                .build()
-                .map_err(to_rb_err)?,
-            None => FileIOBuilder::new_fs_io().build().map_err(to_rb_err)?,
-        };
-        let catalog = MemoryCatalog::new(file_io, warehouse);
+        let mut props = HashMap::new();
+        if let Some(v) = warehouse {
+            props.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), v);
+        }
+        let catalog = runtime()
+            .block_on(MemoryCatalogBuilder::default().load("memory", props))
+            .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Memory(catalog.into()).into(),
         })
@@ -93,16 +95,18 @@ impl RbCatalog {
         uri: String,
         warehouse: Option<String>,
         props: HashMap<String, String>,
-    ) -> Self {
-        let config = RestCatalogConfig::builder()
-            .uri(uri)
-            .warehouse_opt(warehouse)
-            .props(props)
-            .build();
-        let catalog = RestCatalog::new(config);
-        Self {
-            catalog: RbCatalogType::Rest(catalog.into()).into(),
+    ) -> RbResult<Self> {
+        let mut props = props;
+        props.insert(REST_CATALOG_PROP_URI.to_string(), uri);
+        if let Some(v) = warehouse {
+            props.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), v);
         }
+        let catalog = runtime()
+            .block_on(RestCatalogBuilder::default().load("rest", props))
+            .map_err(to_rb_err)?;
+        Ok(Self {
+            catalog: RbCatalogType::Rest(catalog.into()).into(),
+        })
     }
 
     #[cfg(feature = "sql")]

data/ext/iceberg/src/lib.rs

@@ -122,6 +122,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
 
     let class = module.define_class("RbTableScan", ruby.class_object())?;
     class.define_method("plan_files", method!(RbTableScan::plan_files, 0))?;
+    class.define_method("snapshot", method!(RbTableScan::snapshot, 0))?;
 
     Ok(())
 }

data/ext/iceberg/src/scan.rs

@@ -1,11 +1,12 @@
 use futures::TryStreamExt;
 use iceberg::scan::TableScan;
-use magnus::{RArray, Ruby};
+use magnus::{RArray, Ruby, Value};
 use std::cell::RefCell;
 
 use crate::RbResult;
 use crate::error::to_rb_err;
 use crate::runtime::runtime;
+use crate::utils::rb_snapshot;
 
 #[magnus::wrap(class = "Iceberg::RbTableScan")]
 pub struct RbTableScan {
@@ -44,4 +45,11 @@ impl RbTableScan {
         }
         Ok(files)
     }
+
+    pub fn snapshot(ruby: &Ruby, rb_self: &Self) -> RbResult<Option<Value>> {
+        match rb_self.scan.borrow().snapshot() {
+            Some(s) => Ok(Some(rb_snapshot(ruby, s)?)),
+            None => Ok(None),
+        }
+    }
 }

data/ext/iceberg/src/table.rs

@@ -72,6 +72,7 @@ impl RbTable {
         let parquet_writer_builder = ParquetWriterBuilder::new(
             WriterProperties::default(),
             table.metadata().current_schema().clone(),
+            None,
             table.file_io().clone(),
             location_generator.clone(),
             file_name_generator.clone(),
@@ -145,21 +146,21 @@ impl RbTable {
     pub fn schemas(ruby: &Ruby, rb_self: &Self) -> RbResult<RArray> {
         let schemas = ruby.ary_new();
         for s in rb_self.table.borrow().metadata().schemas_iter() {
-            schemas.push(rb_schema(s)?)?;
+            schemas.push(rb_schema(ruby, s)?)?;
         }
         Ok(schemas)
     }
 
-    pub fn schema_by_id(&self, schema_id: i32) -> RbResult<Option<Value>> {
-        let schema = match self.table.borrow().metadata().schema_by_id(schema_id) {
-            Some(s) => Some(rb_schema(s)?),
+    pub fn schema_by_id(ruby: &Ruby, rb_self: &Self, schema_id: i32) -> RbResult<Option<Value>> {
+        let schema = match rb_self.table.borrow().metadata().schema_by_id(schema_id) {
+            Some(s) => Some(rb_schema(ruby, s)?),
             None => None,
         };
         Ok(schema)
     }
 
-    pub fn current_schema(&self) -> RbResult<Value> {
-        rb_schema(self.table.borrow().metadata().current_schema())
+    pub fn current_schema(ruby: &Ruby, rb_self: &Self) -> RbResult<Value> {
+        rb_schema(ruby, rb_self.table.borrow().metadata().current_schema())
     }
 
     pub fn current_schema_id(&self) -> i32 {
@@ -198,14 +199,23 @@ impl RbTable {
     pub fn snapshots(ruby: &Ruby, rb_self: &Self) -> RbResult<RArray> {
         let snapshots = ruby.ary_new();
         for s in rb_self.table.borrow().metadata().snapshots() {
-            snapshots.push(rb_snapshot(s)?)?;
+            snapshots.push(rb_snapshot(ruby, s)?)?;
         }
         Ok(snapshots)
     }
 
-    pub fn snapshot_by_id(&self, snapshot_id: i64) -> RbResult<Option<Value>> {
-        let snapshot = match self.table.borrow().metadata().snapshot_by_id(snapshot_id) {
-            Some(s) => Some(rb_snapshot(s)?),
+    pub fn snapshot_by_id(
+        ruby: &Ruby,
+        rb_self: &Self,
+        snapshot_id: i64,
+    ) -> RbResult<Option<Value>> {
+        let snapshot = match rb_self
+            .table
+            .borrow()
+            .metadata()
+            .snapshot_by_id(snapshot_id)
+        {
+            Some(s) => Some(rb_snapshot(ruby, s)?),
             None => None,
         };
         Ok(snapshot)
@@ -236,9 +246,9 @@ impl RbTable {
         Ok(metadata_logs)
     }
 
-    pub fn current_snapshot(&self) -> RbResult<Option<Value>> {
-        let snapshot = match self.table.borrow().metadata().current_snapshot() {
-            Some(s) => Some(rb_snapshot(s)?),
+    pub fn current_snapshot(ruby: &Ruby, rb_self: &Self) -> RbResult<Option<Value>> {
+        let snapshot = match rb_self.table.borrow().metadata().current_snapshot() {
+            Some(s) => Some(rb_snapshot(ruby, s)?),
             None => None,
         };
         Ok(snapshot)
@@ -248,9 +258,18 @@ impl RbTable {
         self.table.borrow().metadata().current_snapshot_id()
     }
 
-    pub fn snapshot_for_ref(&self, ref_name: String) -> RbResult<Option<Value>> {
-        let snapshot = match self.table.borrow().metadata().snapshot_for_ref(&ref_name) {
-            Some(s) => Some(rb_snapshot(s)?),
+    pub fn snapshot_for_ref(
+        ruby: &Ruby,
+        rb_self: &Self,
+        ref_name: String,
+    ) -> RbResult<Option<Value>> {
+        let snapshot = match rb_self
+            .table
+            .borrow()
+            .metadata()
+            .snapshot_for_ref(&ref_name)
+        {
+            Some(s) => Some(rb_snapshot(ruby, s)?),
             None => None,
         };
         Ok(snapshot)

data/ext/iceberg/src/utils.rs

@@ -146,8 +146,7 @@ fn default_value(ob: Value, field_type: &Type) -> RbResult<Option<Literal>> {
     Ok(Some(lit))
 }
 
-pub fn rb_schema(schema: &Schema) -> RbResult<Value> {
-    let ruby = Ruby::get().unwrap();
+pub fn rb_schema(ruby: &Ruby, schema: &Schema) -> RbResult<Value> {
     let fields = ruby.ary_new();
     for f in schema.as_struct().fields() {
         let field = ruby.hash_new();
@@ -182,10 +181,10 @@ pub fn rb_schema(schema: &Schema) -> RbResult<Value> {
 
         field.aset(ruby.to_symbol("required"), f.required)?;
 
-        let initial_default = f.initial_default.as_ref().map(rb_literal);
+        let initial_default = f.initial_default.as_ref().map(|v| rb_literal(ruby, v));
         field.aset(ruby.to_symbol("initial_default"), initial_default)?;
 
-        let write_default = f.write_default.as_ref().map(rb_literal);
+        let write_default = f.write_default.as_ref().map(|v| rb_literal(ruby, v));
         field.aset(ruby.to_symbol("write_default"), write_default)?;
 
         field.aset(
@@ -205,8 +204,20 @@ pub fn rb_schema(schema: &Schema) -> RbResult<Value> {
         .funcall("new", (fields, kwargs!("schema_id" => schema_id)))
 }
 
-pub fn rb_snapshot(_snapshot: &Snapshot) -> RbResult<Value> {
-    todo!();
+pub fn rb_snapshot(ruby: &Ruby, snapshot: &Snapshot) -> RbResult<Value> {
+    let rb_snapshot = ruby.hash_new();
+    rb_snapshot.aset(ruby.to_symbol("snapshot_id"), snapshot.snapshot_id())?;
+    rb_snapshot.aset(
+        ruby.to_symbol("parent_snapshot_id"),
+        snapshot.parent_snapshot_id(),
+    )?;
+    rb_snapshot.aset(
+        ruby.to_symbol("sequence_number"),
+        snapshot.sequence_number(),
+    )?;
+    rb_snapshot.aset(ruby.to_symbol("manifest_list"), snapshot.manifest_list())?;
+    rb_snapshot.aset(ruby.to_symbol("schema_id"), snapshot.schema_id())?;
+    Ok(rb_snapshot.as_value())
 }
 
 pub fn rb_partition_spec(_partition_spec: &PartitionSpec) -> RbResult<Value> {
@@ -227,15 +238,14 @@ pub fn rb_partition_statistics_file(
     todo!();
 }
 
-pub fn rb_literal(literal: &Literal) -> Value {
-    let ruby = Ruby::get().unwrap();
+pub fn rb_literal(ruby: &Ruby, literal: &Literal) -> Value {
     match literal {
         Literal::Primitive(pl) => match pl {
-            PrimitiveLiteral::Boolean(v) => v.into_value_with(&ruby),
-            PrimitiveLiteral::Int(v) => v.into_value_with(&ruby),
-            PrimitiveLiteral::Long(v) => v.into_value_with(&ruby),
-            PrimitiveLiteral::Float(v) => v.into_value_with(&ruby),
-            PrimitiveLiteral::Double(v) => v.into_value_with(&ruby),
+            PrimitiveLiteral::Boolean(v) => v.into_value_with(ruby),
+            PrimitiveLiteral::Int(v) => v.into_value_with(ruby),
+            PrimitiveLiteral::Long(v) => v.into_value_with(ruby),
+            PrimitiveLiteral::Float(v) => v.into_value_with(ruby),
+            PrimitiveLiteral::Double(v) => v.into_value_with(ruby),
             PrimitiveLiteral::String(v) => ruby.str_new(v).as_value(),
             PrimitiveLiteral::Binary(v) => ruby.str_from_slice(v).as_value(),
             _ => todo!(),

data/lib/iceberg/table.rb

@@ -83,14 +83,28 @@ module Iceberg
       @table.properties
     end
 
-    def to_polars(snapshot_id: nil, storage_options: nil)
+    def scan(snapshot_id: nil)
+      TableScan.new(@table.scan(snapshot_id), self)
+    end
+
+    def to_polars(snapshot_id: nil, storage_options: nil, _schema_changes: false)
       require "polars-df"
 
-      files = @table.scan(snapshot_id).plan_files
+      # TODO always take this path in 0.2.0
+      if _schema_changes
+        return Polars.scan_iceberg(self, snapshot_id:, storage_options:)
+      end
+
+      scan = scan(snapshot_id:)
+      files = scan.plan_files
+
       if files.empty?
+        snapshot = scan.snapshot
+        scan_schema = snapshot ? schema_by_id(snapshot[:schema_id]) : current_schema
+
         # TODO improve
         schema =
-          current_schema.fields.to_h do |field|
+          scan_schema.fields.to_h do |field|
             dtype =
               case field[:type]
               when "int"
@@ -121,16 +135,12 @@ module Iceberg
             .to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
         ]
 
-        Polars.scan_parquet(
-          sources,
+        scan_options = {
           storage_options: storage_options,
-          # TODO
-          # cast_options: Polars::ScanCastOptions._default_iceberg,
-          # allow_missing_columns: true,
-          # extra_columns: "ignore",
-          # _column_mapping: column_mapping,
-          _deletion_files: deletion_files
-        )
+          _deletion_files: deletion_files,
+        }
+
+        Polars.scan_parquet(sources, **scan_options)
       end
     end
 

data/lib/iceberg/table_scan.rb

@@ -0,0 +1,18 @@
+module Iceberg
+  class TableScan
+    attr_reader :table
+
+    def initialize(scan, table)
+      @scan = scan
+      @table = table
+    end
+
+    def plan_files
+      @scan.plan_files
+    end
+
+    def snapshot
+      @scan.snapshot
+    end
+  end
+end

data/lib/iceberg/version.rb

@@ -1,3 +1,3 @@
 module Iceberg
-  VERSION = "0.10.1"
+  VERSION = "0.10.3"
 end

data/lib/iceberg.rb

@@ -9,6 +9,7 @@ end
 require_relative "iceberg/catalog"
 require_relative "iceberg/schema"
 require_relative "iceberg/table"
+require_relative "iceberg/table_scan"
 require_relative "iceberg/static_table"
 require_relative "iceberg/table_definition"
 require_relative "iceberg/version"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iceberg
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.10.3
 platform: ruby
 authors:
 - Andrew Kane
@@ -55,6 +55,7 @@ files:
 - lib/iceberg/static_table.rb
 - lib/iceberg/table.rb
 - lib/iceberg/table_definition.rb
+- lib/iceberg/table_scan.rb
 - lib/iceberg/version.rb
 homepage: https://github.com/ankane/iceberg-ruby
 licenses: