deltalake-rb 0.1.1 → 0.1.3

data/lib/deltalake.rb

@@ -7,6 +7,7 @@ end
 
 # stdlib
 require "json"
+require "time"
 
 # modules
 require_relative "deltalake/field"
@@ -14,7 +15,9 @@ require_relative "deltalake/metadata"
 require_relative "deltalake/schema"
 require_relative "deltalake/table"
 require_relative "deltalake/table_alterer"
+require_relative "deltalake/table_merger"
 require_relative "deltalake/table_optimizer"
+require_relative "deltalake/utils"
 require_relative "deltalake/version"
 
 module DeltaLake
@@ -38,6 +41,28 @@ module DeltaLake
       :reader_features
     )
 
+  CommitProperties =
+    Struct.new(
+      :custom_metadata,
+      :max_commit_retries,
+      # TODO
+      # :app_transactions,
+      keyword_init: true
+    )
+
+  PostCommitHookProperties =
+    Struct.new(
+      :create_checkpoint,
+      :cleanup_expired_logs,
+      keyword_init: true
+    )
+
+  class ArrowArrayStream
+    def arrow_c_stream
+      self
+    end
+  end
+
   class << self
     def write(
       table_or_uri,
@@ -50,7 +75,10 @@ module DeltaLake
       schema_mode: nil,
       storage_options: nil,
       predicate: nil,
-      target_file_size: nil
+      target_file_size: nil,
+      writer_properties: nil,
+      commit_properties: nil,
+      post_commithook_properties: nil
     )
       table, table_uri = try_get_table_and_table_uri(table_or_uri, storage_options)
 
@@ -62,7 +90,7 @@ module DeltaLake
         return
       end
 
-      data = convert_data(data)
+      data = Utils.convert_data(data)
 
       write_deltalake_rust(
         table_uri,
@@ -76,7 +104,10 @@ module DeltaLake
         name,
         description,
         configuration,
-        storage_options
+        storage_options,
+        writer_properties,
+        commit_properties,
+        post_commithook_properties
       )
 
       if table
@@ -107,61 +138,5 @@ module DeltaLake
     rescue TableNotFoundError
       nil
     end
-
-    def convert_data(data)
-      if data.respond_to?(:arrow_c_stream)
-        # TODO convert other object types
-        # should probably move logic to Rust
-        if defined?(Polars::DataFrame) && data.is_a?(Polars::DataFrame)
-          data = convert_polars_data(data)
-        end
-
-        data.arrow_c_stream
-      else
-        raise TypeError, "Only objects implementing the Arrow C stream interface are valid inputs for source."
-      end
-    end
-
-    # unsigned integers are not part of the protocol
-    # https://github.com/delta-io/delta/blob/master/PROTOCOL.md#primitive-types
-    def convert_polars_data(data)
-      new_schema = {}
-      data.schema.each do |k, v|
-        new_type = convert_polars_type(v)
-        new_schema[k] = new_type if new_type
-      end
-
-      if new_schema.any?
-        data.cast(new_schema)
-      else
-        data
-      end
-    end
-
-    def convert_polars_type(t)
-      case t
-      when Polars::UInt8
-        Polars::Int8
-      when Polars::UInt16
-        Polars::Int16
-      when Polars::UInt32
-        Polars::Int32
-      when Polars::UInt64
-        Polars::Int64
-      when Polars::Datetime
-        Polars::Datetime.new("us", t.time_zone) if t.time_unit != "us"
-      when Polars::List
-        inner = convert_polars_type(t.inner)
-        Polars::List.new(inner) if inner
-      when Polars::Array
-        inner = convert_polars_type(t.inner)
-        Polars::Array.new(t.inner, t.width) if inner
-      when Polars::Struct
-        if t.fields.any? { |f| convert_polars_type(f.dtype) }
-          fields = t.fields.map { |f| Polars::Field.new(f.name, convert_polars_type(f.dtype) || f.dtype) }
-          Polars::Struct.new(fields)
-        end
-      end
-    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deltalake-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-23 00:00:00.000000000 Z
+date: 2024-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -39,7 +39,9 @@ files:
 - ext/deltalake/Cargo.toml
 - ext/deltalake/extconf.rb
 - ext/deltalake/src/error.rs
+- ext/deltalake/src/features.rs
 - ext/deltalake/src/lib.rs
+- ext/deltalake/src/merge.rs
 - ext/deltalake/src/schema.rs
 - ext/deltalake/src/utils.rs
 - lib/deltalake-rb.rb
@@ -49,7 +51,9 @@ files:
 - lib/deltalake/schema.rb
 - lib/deltalake/table.rb
 - lib/deltalake/table_alterer.rb
+- lib/deltalake/table_merger.rb
 - lib/deltalake/table_optimizer.rb
+- lib/deltalake/utils.rb
 - lib/deltalake/version.rb
 homepage: https://github.com/ankane/delta-ruby
 licenses: