deltalake-rb 0.1.1 → 0.1.3

data/README.md

@@ -21,15 +21,15 @@ It can take 5-10 minutes to compile the gem.
 Write data
 
 ```ruby
-df = Polars::DataFrame.new({"a" => [1, 2], "b" => [3.0, 4.0]})
-DeltaLake.write("./data/delta", df)
+df = Polars::DataFrame.new({"id" => [1, 2], "value" => [3.0, 4.0]})
+DeltaLake.write("./events", df)
 ```
 
 Load a table
 
 ```ruby
-dt = DeltaLake::Table.new("./data/delta")
-df2 = dt.to_polars
+dt = DeltaLake::Table.new("./events")
+df = dt.to_polars
 ```
 
 Get a lazy frame
@@ -41,31 +41,31 @@ lf = dt.to_polars(eager: false)
 Append rows
 
 ```ruby
-DeltaLake.write("./data/delta", df, mode: "append")
+DeltaLake.write("./events", df, mode: "append")
 ```
 
 Overwrite a table
 
 ```ruby
-DeltaLake.write("./data/delta", df, mode: "overwrite")
+DeltaLake.write("./events", df, mode: "overwrite")
 ```
 
 Add a constraint
 
 ```ruby
-dt.alter.add_constraint({"a_gt_0" => "a > 0"})
+dt.alter.add_constraint({"id_gt_0" => "id > 0"})
 ```
 
 Drop a constraint
 
 ```ruby
-dt.alter.drop_constraint("a_gt_0")
+dt.alter.drop_constraint("id_gt_0")
 ```
 
 Delete rows
 
 ```ruby
-dt.delete("a > 1")
+dt.delete("id > 1")
 ```
 
 Vacuum
@@ -83,13 +83,13 @@ dt.optimize.compact
 Colocate similar data in the same files
 
 ```ruby
-dt.optimize.z_order(["a"])
+dt.optimize.z_order(["category"])
 ```
 
 Load a previous version of a table
 
 ```ruby
-dt = DeltaLake::Table.new("./data/delta", version: 1)
+dt = DeltaLake::Table.new("./events", version: 1)
 # or
 dt.load_as_version(1)
 ```

data/ext/deltalake/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake"
-version = "0.1.1"
+version = "0.1.3"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -11,10 +11,11 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "52", features = ["ffi"] }
-arrow-schema = { version = "52", features = ["serde"] }
+arrow = { version = "53", features = ["ffi"] }
+arrow-schema = { version = "53", features = ["serde"] }
 chrono = "0.4"
-deltalake = { version = "=0.21.0", features = ["datafusion", "s3"] }
+delta_kernel = "0.4"
+deltalake = { version = "=0.22.3", features = ["azure", "datafusion", "gcs", "s3"] }
 futures = "0.3"
 magnus = "0.7"
 num_cpus = "1"

data/ext/deltalake/src/error.rs

@@ -1,6 +1,7 @@
 use arrow_schema::ArrowError;
+use deltalake::protocol::ProtocolError;
 use deltalake::{errors::DeltaTableError, ObjectStoreError};
-use magnus::{exception, Error, Module, RModule, Ruby};
+use magnus::{exception, Error as RbErr, Module, RModule, Ruby};
 use std::borrow::Cow;
 
 macro_rules! create_exception {
@@ -8,7 +9,7 @@ macro_rules! create_exception {
         pub struct $type {}
 
         impl $type {
-            pub fn new_err<T>(message: T) -> Error
+            pub fn new_err<T>(message: T) -> RbErr
             where
                 T: Into<Cow<'static, str>>,
             {
@@ -19,7 +20,7 @@ macro_rules! create_exception {
                     .unwrap()
                     .const_get($name)
                     .unwrap();
-                Error::new(class, message)
+                RbErr::new(class, message)
             }
         }
     };
@@ -31,7 +32,7 @@ create_exception!(DeltaProtocolError, "DeltaProtocolError");
 create_exception!(CommitFailedError, "CommitFailedError");
 create_exception!(SchemaMismatchError, "SchemaMismatchError");
 
-fn inner_to_rb_err(err: DeltaTableError) -> Error {
+fn inner_to_rb_err(err: DeltaTableError) -> RbErr {
     match err {
         DeltaTableError::NotATable(msg) => TableNotFoundError::new_err(msg),
         DeltaTableError::InvalidTableLocation(msg) => TableNotFoundError::new_err(msg),
@@ -48,7 +49,7 @@ fn inner_to_rb_err(err: DeltaTableError) -> Error {
 
         // ruby exceptions
         DeltaTableError::ObjectStore { source } => object_store_to_rb(source),
-        DeltaTableError::Io { source } => Error::new(exception::io_error(), source.to_string()),
+        DeltaTableError::Io { source } => RbIOError::new_err(source.to_string()),
 
         DeltaTableError::Arrow { source } => arrow_to_rb(source),
 
@@ -56,31 +57,50 @@ fn inner_to_rb_err(err: DeltaTableError) -> Error {
     }
 }
 
-fn object_store_to_rb(err: ObjectStoreError) -> Error {
+fn object_store_to_rb(err: ObjectStoreError) -> RbErr {
     match err {
-        ObjectStoreError::NotFound { .. } => Error::new(exception::io_error(), err.to_string()),
+        ObjectStoreError::NotFound { .. } => RbIOError::new_err(err.to_string()),
         ObjectStoreError::Generic { source, .. }
             if source.to_string().contains("AWS_S3_ALLOW_UNSAFE_RENAME") =>
         {
             DeltaProtocolError::new_err(source.to_string())
         }
-        _ => Error::new(exception::io_error(), err.to_string()),
+        _ => RbIOError::new_err(err.to_string()),
     }
 }
 
-fn arrow_to_rb(err: ArrowError) -> Error {
+fn arrow_to_rb(err: ArrowError) -> RbErr {
     match err {
-        ArrowError::IoError(msg, _) => Error::new(exception::io_error(), msg),
-        ArrowError::DivideByZero => Error::new(exception::arg_error(), "division by zero"),
-        ArrowError::InvalidArgumentError(msg) => Error::new(exception::arg_error(), msg),
-        ArrowError::NotYetImplemented(msg) => Error::new(exception::not_imp_error(), msg),
+        ArrowError::IoError(msg, _) => RbIOError::new_err(msg),
+        ArrowError::DivideByZero => RbValueError::new_err("division by zero"),
+        ArrowError::InvalidArgumentError(msg) => RbValueError::new_err(msg),
+        ArrowError::NotYetImplemented(msg) => RbNotImplementedError::new_err(msg),
         ArrowError::SchemaError(msg) => SchemaMismatchError::new_err(msg),
-        other => Error::new(exception::runtime_error(), other.to_string()),
+        other => RbException::new_err(other.to_string()),
+    }
+}
+
+fn checkpoint_to_rb(err: ProtocolError) -> RbErr {
+    match err {
+        ProtocolError::Arrow { source } => arrow_to_rb(source),
+        ProtocolError::ObjectStore { source } => object_store_to_rb(source),
+        ProtocolError::EndOfLog => DeltaProtocolError::new_err("End of log"),
+        ProtocolError::NoMetaData => DeltaProtocolError::new_err("Table metadata missing"),
+        ProtocolError::CheckpointNotFound => DeltaProtocolError::new_err(err.to_string()),
+        ProtocolError::InvalidField(err) => RbValueError::new_err(err),
+        ProtocolError::InvalidRow(err) => RbValueError::new_err(err),
+        ProtocolError::InvalidDeletionVectorStorageType(err) => RbValueError::new_err(err),
+        ProtocolError::SerializeOperation { source } => RbValueError::new_err(source.to_string()),
+        ProtocolError::ParquetParseError { source } => RbIOError::new_err(source.to_string()),
+        ProtocolError::IO { source } => RbIOError::new_err(source.to_string()),
+        ProtocolError::Generic(msg) => DeltaError::new_err(msg),
+        ProtocolError::Kernel { source } => DeltaError::new_err(source.to_string()),
     }
 }
 
 pub enum RubyError {
     DeltaTable(DeltaTableError),
+    Protocol(ProtocolError),
 }
 
 impl From<DeltaTableError> for RubyError {
@@ -89,10 +109,37 @@ impl From<DeltaTableError> for RubyError {
     }
 }
 
-impl From<RubyError> for Error {
+impl From<ProtocolError> for RubyError {
+    fn from(err: ProtocolError) -> Self {
+        RubyError::Protocol(err)
+    }
+}
+
+impl From<RubyError> for RbErr {
     fn from(value: RubyError) -> Self {
         match value {
             RubyError::DeltaTable(err) => inner_to_rb_err(err),
+            RubyError::Protocol(err) => checkpoint_to_rb(err),
         }
     }
 }
+
+macro_rules! create_builtin_exception {
+    ($type:ident, $class:expr) => {
+        pub struct $type {}
+
+        impl $type {
+            pub fn new_err<T>(message: T) -> RbErr
+            where
+                T: Into<Cow<'static, str>>,
+            {
+                RbErr::new($class, message)
+            }
+        }
+    };
+}
+
+create_builtin_exception!(RbException, exception::runtime_error());
+create_builtin_exception!(RbIOError, exception::io_error());
+create_builtin_exception!(RbNotImplementedError, exception::not_imp_error());
+create_builtin_exception!(RbValueError, exception::arg_error());

data/ext/deltalake/src/features.rs

@@ -0,0 +1,67 @@
+use crate::{RbResult, RbValueError};
+use deltalake::kernel::TableFeatures as KernelTableFeatures;
+use magnus::{prelude::*, TryConvert, Value};
+
+/// High level table features
+#[derive(Clone)]
+pub enum TableFeatures {
+    /// Mapping of one column to another
+    ColumnMapping,
+    /// Deletion vectors for merge, update, delete
+    DeletionVectors,
+    /// timestamps without timezone support
+    TimestampWithoutTimezone,
+    /// version 2 of checkpointing
+    V2Checkpoint,
+    /// Append Only Tables
+    AppendOnly,
+    /// Table invariants
+    Invariants,
+    /// Check constraints on columns
+    CheckConstraints,
+    /// CDF on a table
+    ChangeDataFeed,
+    /// Columns with generated values
+    GeneratedColumns,
+    /// ID Columns
+    IdentityColumns,
+    /// Row tracking on tables
+    RowTracking,
+    /// domain specific metadata
+    DomainMetadata,
+    /// Iceberg compatibility support
+    IcebergCompatV1,
+}
+
+impl From<TableFeatures> for KernelTableFeatures {
+    fn from(value: TableFeatures) -> Self {
+        match value {
+            TableFeatures::ColumnMapping => KernelTableFeatures::ColumnMapping,
+            TableFeatures::DeletionVectors => KernelTableFeatures::DeletionVectors,
+            TableFeatures::TimestampWithoutTimezone => {
+                KernelTableFeatures::TimestampWithoutTimezone
+            }
+            TableFeatures::V2Checkpoint => KernelTableFeatures::V2Checkpoint,
+            TableFeatures::AppendOnly => KernelTableFeatures::AppendOnly,
+            TableFeatures::Invariants => KernelTableFeatures::Invariants,
+            TableFeatures::CheckConstraints => KernelTableFeatures::CheckConstraints,
+            TableFeatures::ChangeDataFeed => KernelTableFeatures::ChangeDataFeed,
+            TableFeatures::GeneratedColumns => KernelTableFeatures::GeneratedColumns,
+            TableFeatures::IdentityColumns => KernelTableFeatures::IdentityColumns,
+            TableFeatures::RowTracking => KernelTableFeatures::RowTracking,
+            TableFeatures::DomainMetadata => KernelTableFeatures::DomainMetadata,
+            TableFeatures::IcebergCompatV1 => KernelTableFeatures::IcebergCompatV1,
+        }
+    }
+}
+
+impl TryConvert for TableFeatures {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        // TODO add more features
+        let feature = match unsafe { val.to_r_string()?.as_str()? } {
+            "append_only" => TableFeatures::AppendOnly,
+            _ => return Err(RbValueError::new_err("Invalid feature")),
+        };
+        Ok(feature)
+    }
+}