iceberg 0.10.2 → 0.10.3

data/README.md

@@ -82,9 +82,7 @@ Iceberg::MemoryCatalog.new(
 )
 ```
 
-## Reference
-
-### Namespaces
+## Namespaces
 
 List namespaces
 
@@ -122,7 +120,7 @@ Drop a namespace
 catalog.drop_namespace("main")
 ```
 
-### Tables
+## Tables
 
 List tables
 
@@ -169,7 +167,7 @@ Drop a table
 catalog.drop_table("main.events")
 ```
 
-### Static Tables
+## Static Tables
 
 Load a static table
 

data/ext/iceberg/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "iceberg"
-version = "0.10.2"
+version = "0.10.3"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2024"
-rust-version = "1.86"
+rust-version = "1.87"
 publish = false
 
 [lib]
@@ -13,13 +13,13 @@ crate-type = ["cdylib"]
 [dependencies]
 arrow-array = { version = "55", features = ["ffi"] }
 arrow-schema = "55"
-datafusion = { version = "47", optional = true }
+datafusion = { version = "48", optional = true }
 futures = "0.3"
-iceberg = "0.6"
-iceberg-catalog-glue = { version = "0.6", optional = true }
-iceberg-catalog-rest = { version = "0.6", optional = true }
-iceberg-catalog-sql = { version = "0.6", optional = true }
-iceberg-datafusion = { version = "0.6", optional = true }
+iceberg = "0.7"
+iceberg-catalog-glue = { version = "0.7", optional = true }
+iceberg-catalog-rest = { version = "0.7", optional = true }
+iceberg-catalog-sql = { version = "0.7", optional = true }
+iceberg-datafusion = { version = "0.7", optional = true }
 magnus = "0.8"
 parquet = "55"
 sqlx = { version = "0.8", features = ["postgres", "runtime-tokio", "sqlite"], default-features = false, optional = true }

data/ext/iceberg/src/catalog.rs

@@ -1,12 +1,15 @@
 #[cfg(feature = "datafusion")]
 use datafusion::execution::context::SessionContext;
-use iceberg::io::{FileIO, FileIOBuilder};
+use iceberg::io::FileIO;
+use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder};
 use iceberg::spec::Schema;
-use iceberg::{Catalog, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
+use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, TableCreation, TableIdent};
 #[cfg(feature = "glue")]
-use iceberg_catalog_glue::{GlueCatalog, GlueCatalogConfig};
+use iceberg_catalog_glue::{GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder};
 #[cfg(feature = "rest")]
-use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig};
+use iceberg_catalog_rest::{
+    REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RestCatalog, RestCatalogBuilder,
+};
 #[cfg(feature = "sql")]
 use iceberg_catalog_sql::{SqlBindStyle, SqlCatalog, SqlCatalogConfig};
 #[cfg(feature = "datafusion")]
@@ -65,9 +68,9 @@ pub struct RbCatalog {
 impl RbCatalog {
     #[cfg(feature = "glue")]
     pub fn new_glue(warehouse: String) -> RbResult<Self> {
-        let config = GlueCatalogConfig::builder().warehouse(warehouse).build();
+        let props = HashMap::from([(GLUE_CATALOG_PROP_WAREHOUSE.to_string(), warehouse)]);
         let catalog = runtime()
-            .block_on(GlueCatalog::new(config))
+            .block_on(GlueCatalogBuilder::default().load("glue", props))
             .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Glue(catalog.into()).into(),
@@ -75,14 +78,13 @@ impl RbCatalog {
     }
 
     pub fn new_memory(warehouse: Option<String>) -> RbResult<Self> {
-        let file_io = match warehouse {
-            Some(ref v) => FileIO::from_path(v)
-                .map_err(to_rb_err)?
-                .build()
-                .map_err(to_rb_err)?,
-            None => FileIOBuilder::new_fs_io().build().map_err(to_rb_err)?,
-        };
-        let catalog = MemoryCatalog::new(file_io, warehouse);
+        let mut props = HashMap::new();
+        if let Some(v) = warehouse {
+            props.insert(MEMORY_CATALOG_WAREHOUSE.to_string(), v);
+        }
+        let catalog = runtime()
+            .block_on(MemoryCatalogBuilder::default().load("memory", props))
+            .map_err(to_rb_err)?;
         Ok(Self {
             catalog: RbCatalogType::Memory(catalog.into()).into(),
         })
@@ -93,16 +95,18 @@ impl RbCatalog {
         uri: String,
         warehouse: Option<String>,
         props: HashMap<String, String>,
-    ) -> Self {
-        let config = RestCatalogConfig::builder()
-            .uri(uri)
-            .warehouse_opt(warehouse)
-            .props(props)
-            .build();
-        let catalog = RestCatalog::new(config);
-        Self {
-            catalog: RbCatalogType::Rest(catalog.into()).into(),
+    ) -> RbResult<Self> {
+        let mut props = props;
+        props.insert(REST_CATALOG_PROP_URI.to_string(), uri);
+        if let Some(v) = warehouse {
+            props.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), v);
         }
+        let catalog = runtime()
+            .block_on(RestCatalogBuilder::default().load("rest", props))
+            .map_err(to_rb_err)?;
+        Ok(Self {
+            catalog: RbCatalogType::Rest(catalog.into()).into(),
+        })
     }
 
     #[cfg(feature = "sql")]

data/ext/iceberg/src/table.rs

@@ -72,6 +72,7 @@ impl RbTable {
         let parquet_writer_builder = ParquetWriterBuilder::new(
             WriterProperties::default(),
             table.metadata().current_schema().clone(),
+            None,
             table.file_io().clone(),
             location_generator.clone(),
             file_name_generator.clone(),

data/lib/iceberg/table.rb

@@ -83,15 +83,28 @@ module Iceberg
       @table.properties
     end
 
-    def to_polars(snapshot_id: nil, storage_options: nil)
+    def scan(snapshot_id: nil)
+      TableScan.new(@table.scan(snapshot_id), self)
+    end
+
+    def to_polars(snapshot_id: nil, storage_options: nil, _schema_changes: false)
       require "polars-df"
 
-      files = @table.scan(snapshot_id).plan_files
+      # TODO always take this path in 0.2.0
+      if _schema_changes
+        return Polars.scan_iceberg(self, snapshot_id:, storage_options:)
+      end
+
+      scan = scan(snapshot_id:)
+      files = scan.plan_files
+
       if files.empty?
+        snapshot = scan.snapshot
+        scan_schema = snapshot ? schema_by_id(snapshot[:schema_id]) : current_schema
+
         # TODO improve
         schema =
-          # TODO use schema from snapshot_id
-          current_schema.fields.to_h do |field|
+          scan_schema.fields.to_h do |field|
             dtype =
               case field[:type]
               when "int"
@@ -122,16 +135,12 @@ module Iceberg
             .to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
         ]
 
-        Polars.scan_parquet(
-          sources,
+        scan_options = {
           storage_options: storage_options,
-          # TODO
-          # cast_options: Polars::ScanCastOptions._default_iceberg,
-          # allow_missing_columns: true,
-          # extra_columns: "ignore",
-          # _column_mapping: column_mapping,
-          _deletion_files: deletion_files
-        )
+          _deletion_files: deletion_files,
+        }
+
+        Polars.scan_parquet(sources, **scan_options)
       end
     end
 

data/lib/iceberg/table_scan.rb

@@ -0,0 +1,18 @@
+module Iceberg
+  class TableScan
+    attr_reader :table
+
+    def initialize(scan, table)
+      @scan = scan
+      @table = table
+    end
+
+    def plan_files
+      @scan.plan_files
+    end
+
+    def snapshot
+      @scan.snapshot
+    end
+  end
+end

data/lib/iceberg/version.rb

@@ -1,3 +1,3 @@
 module Iceberg
-  VERSION = "0.10.2"
+  VERSION = "0.10.3"
 end

data/lib/iceberg.rb

@@ -9,6 +9,7 @@ end
 require_relative "iceberg/catalog"
 require_relative "iceberg/schema"
 require_relative "iceberg/table"
+require_relative "iceberg/table_scan"
 require_relative "iceberg/static_table"
 require_relative "iceberg/table_definition"
 require_relative "iceberg/version"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iceberg
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.10.3
 platform: ruby
 authors:
 - Andrew Kane
@@ -55,6 +55,7 @@ files:
 - lib/iceberg/static_table.rb
 - lib/iceberg/table.rb
 - lib/iceberg/table_definition.rb
+- lib/iceberg/table_scan.rb
 - lib/iceberg/version.rb
 homepage: https://github.com/ankane/iceberg-ruby
 licenses: