ducklake 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 451f6059faf0e7c451599b218bb9dafbbb8b80a2b54416833a5dafb85a0d1a9d
-  data.tar.gz: a1c6c7825da3dd5a077705c1c3f106a0cc1d75bc2e60b17ccb5099f98a3bdf1a
+  metadata.gz: 98460f8bbd24007057521ad2749899aebd4e689c0183ab98551e7c0e9d7b8ffe
+  data.tar.gz: 726ea12c95079c99999b9415fb915e05818548bcab38647413eee44018c2a7f3
 SHA512:
-  metadata.gz: 9359cf3aaf06ed4398b9bc0ff35489c0e50008cd8d3426a787339d33f541a6bce8e3a1eca4a2b88b46852545895b591d08613f88ad26902aa4d41cf8b842bc93
-  data.tar.gz: 5bd9818d1b928aa2c0a11301c3455b89f8a4fc99912c6d1b8e52fdd151088adcc331b0ca01169df08412e151841f2f6b50f4c7b85b909ca558d6c21f0ffebd56
+  metadata.gz: 772924fb6262d3a3373eb2e92431b55dc73f81869e2aaf1a92746831c1d784f669a9e82a03a8c4d2538bda7c7980e940cd29f65c068ca996ea71a55c19d37dbd
+  data.tar.gz: c57170ee010ea949d4f8c250f3ff4fcc05a3153b1e25d9412ba12cd53e4ef7e866f088e2956886948757e0586f4005087e50c89fff1295404ce4c01baae79ce1

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.1.3 (2025-09-23)
+
+- Added `current_snapshot` and `last_committed_snapshot` methods
+- Added `rewrite_data_files` and `delete_orphaned_files` methods
+- Added `commit_message` and `commit_author` options to `transaction` method
+- Added `migrate_if_required` option
+- Added experimental support for encryption
+
+## 0.1.2 (2025-08-23)
+
+- Added `transaction` method
+
 ## 0.1.1 (2025-08-18)
 
 - Added experimental support for Polars

data/README.md

@@ -154,7 +154,7 @@ Note: This transfers ownership to the data lake, so the file may be deleted as p
 Update data
 
 ```ruby
-ducklake.sql("UPDATE events SET name = ? WHERE id = 1", ["Test", 1])
+ducklake.sql("UPDATE events SET name = ? WHERE id = ?", ["Test", 1])
 ```
 
 Delete data
@@ -163,12 +163,46 @@ Delete data
 ducklake.sql("DELETE * FROM events WHERE id = ?", [1])
 ```
 
+Run multiple statements in a transaction
+
+```ruby
+ducklake.transaction do
+  # ...
+end
+```
+
+Raise `DuckLake::Rollback` to rollback
+
+## Schema Changes
+
 Update the schema
 
 ```ruby
 ducklake.sql("ALTER TABLE events ADD COLUMN active BOOLEAN")
 ```
 
+Set or remove a [partitioning key](https://ducklake.select/docs/stable/duckdb/advanced_features/partitioning)
+
+```ruby
+ducklake.sql("ALTER TABLE events SET PARTITIONED BY (name)")
+# or
+ducklake.sql("ALTER TABLE events RESET PARTITIONED BY")
+```
+
+## Views
+
+Create a view
+
+```ruby
+ducklake.sql("CREATE VIEW events_view AS SELECT * FROM events")
+```
+
+Drop a view
+
+```ruby
+ducklake.sql("DROP VIEW events_view")
+```
+
 ## Snapshots
 
 Get snapshots
@@ -181,7 +215,7 @@ Query the data at a specific snapshot version or time
 
 ```ruby
 ducklake.sql("SELECT * FROM events AT (VERSION => ?)", [3])
-#
+# or
 ducklake.sql("SELECT * FROM events AT (TIMESTAMP => ?)", [Date.today - 7])
 ```
 
@@ -235,9 +269,9 @@ ducklake.set_option("parquet_compression", "zstd", table_name: "events")
 
 ## Read-Only Mode
 
-Note: This feature is experimental
+Note: This feature is experimental and does not prevent the DuckDB engine from writing files via `sql`
 
-Connect to the data lake in read-only mode
+Attach the catalog in read-only mode
 
 ```ruby
 DuckLake::Client.new(read_only: true, ...)
@@ -292,9 +326,19 @@ quoted_file = ducklake.quote("path/to/data.csv")
 ducklake.sql("COPY #{quoted_table} FROM #{quoted_file}")
 ```
 
+## Encryption
+
+Note: This feature is unreleased and must be set when creating the catalog
+
+Encrypt Parquet files
+
+```ruby
+DuckLake::Client.new(encryption: true, ...)
+```
+
 ## Polars
 
-Note: This feature is experimental and does not work on tables with schema changes
+Note: This feature is experimental and does not currently work on tables with schema changes
 
 Query the data with [Ruby Polars](https://github.com/ankane/ruby-polars)
 
@@ -368,10 +412,12 @@ bundle install
 
 # Postgres
 createdb ducklake_ruby_test
+createdb ducklake_ruby_test2
 bundle exec rake test:postgres
 
 # MySQL and MariaDB
 mysqladmin create ducklake_ruby_test
+mysqladmin create ducklake_ruby_test2
 bundle exec rake test:mysql
 
 # SQLite

data/lib/ducklake/client.rb

@@ -8,7 +8,10 @@ module DuckLake
       snapshot_time: nil,
       data_inlining_row_limit: 0,
       create_if_not_exists: false,
-      read_only: false # experimental
+      migrate_if_required: true, # TODO make false in 0.2.0
+      read_only: false, # experimental
+      override_storage_url: false, # experimental
+      encrypted: false # experimental
     )
       catalog_uri = URI.parse(catalog_url)
       storage_uri = URI.parse(storage_url)
@@ -58,10 +61,13 @@ module DuckLake
 
       attach_options = {data_path: storage_url}
       attach_options[:read_only] = true if read_only
+      attach_options[:encrypted] = 1 if encrypted
       attach_options[:snapshot_version] = snapshot_version if !snapshot_version.nil?
       attach_options[:snapshot_time] = snapshot_time if !snapshot_time.nil?
       attach_options[:data_inlining_row_limit] = data_inlining_row_limit if data_inlining_row_limit > 0
       attach_options[:create_if_not_exists] = false unless create_if_not_exists
+      attach_options[:migrate_if_required] = false unless migrate_if_required
+      attach_options[:override_data_path] = true if override_storage_url
 
       @catalog = "ducklake"
       @storage_url = storage_url
@@ -106,6 +112,18 @@ module DuckLake
       execute(sql, params)
     end
 
+    def transaction(commit_message: nil, commit_author: nil)
+      execute("BEGIN")
+      begin
+        yield
+        set_commit_message(commit_message, commit_author) if commit_message || commit_author
+        execute("COMMIT")
+      rescue => e
+        execute("ROLLBACK")
+        raise e unless e.is_a?(Rollback)
+      end
+    end
+
     def attach(alias_, url)
       type = nil
       extension = nil
@@ -151,6 +169,15 @@ module DuckLake
       symbolize_keys result
     end
 
+    # experimental
+    # TODO use keyword arguments or range?
+    def table_changes(table, start_snapshot, end_snapshot)
+      params = [@catalog, "main", table, start_snapshot, end_snapshot]
+      result = execute("SELECT * FROM ducklake_table_changes(?, ?, ?, ?, ?)", params)
+      # only return changes between snapshots
+      symbolize_keys result.reject { |v| v["snapshot_id"] == start_snapshot }
+    end
+
     # TODO more DDL methods?
     def drop_table(table, if_exists: nil)
       execute("DROP TABLE#{" IF EXISTS" if if_exists} #{quote_identifier(table)}")
@@ -162,6 +189,14 @@ module DuckLake
       symbolize_keys execute("SELECT * FROM ducklake_snapshots(?)", [@catalog])
     end
 
+    def current_snapshot
+      execute("SELECT * FROM ducklake_current_snapshot(?)", [@catalog]).rows[0][0]
+    end
+
+    def last_committed_snapshot
+      execute("SELECT * FROM ducklake_last_committed_snapshot(?)", [@catalog]).rows[0][0]
+    end
+
     # https://ducklake.select/docs/stable/duckdb/usage/configuration
     def options
       symbolize_keys execute("SELECT * FROM ducklake_options(?)", [@catalog])
@@ -237,6 +272,55 @@ module DuckLake
       symbolize_keys execute("CALL ducklake_cleanup_old_files(#{args.join(", ")})", params)
     end
 
+    # https://ducklake.select/docs/stable/duckdb/maintenance/cleanup_of_files#cleanup-of-orphaned-files
+    def delete_orphaned_files(cleanup_all: false, older_than: nil, dry_run: false)
+      args = ["?"]
+      params = [@catalog]
+
+      if cleanup_all
+        args << "cleanup_all => ?"
+        params << cleanup_all
+      end
+
+      if !older_than.nil?
+        args << "older_than => ?"
+        params << older_than
+      end
+
+      if dry_run
+        args << "dry_run => ?"
+        params << dry_run
+      end
+
+      symbolize_keys execute("CALL ducklake_delete_orphaned_files(#{args.join(", ")})", params)
+    end
+
+    # https://ducklake.select/docs/stable/duckdb/maintenance/rewrite_data_files
+    def rewrite_data_files(table = nil, delete_threshold: nil)
+      args = ["?"]
+      params = [@catalog]
+
+      if !table.nil?
+        args << "?"
+        params << table
+      end
+
+      if !delete_threshold.nil?
+        args << "delete_threshold => ?"
+        params << delete_threshold
+      end
+
+      execute("CALL ducklake_rewrite_data_files(#{args.join(", ")})", params)
+      nil
+    end
+
+    # experimental
+    # https://ducklake.select/docs/stable/duckdb/maintenance/checkpoint
+    def checkpoint
+      execute("CHECKPOINT")
+      nil
+    end
+
     # https://ducklake.select/docs/stable/duckdb/advanced_features/data_inlining
     def flush_inlined_data(table_name: nil)
       args = ["?"]
@@ -289,15 +373,26 @@ module DuckLake
     end
 
     # experimental
-    # TODO support schema changes
     def polars(table, snapshot_version: nil, snapshot_time: nil)
       files = list_files(table, snapshot_version:, snapshot_time:)
       sources = files.map { |v| v[:data_file] }
+      # TODO support schema changes
+      # column_mapping = [
+      #   "iceberg-column-mapping",
+      #   nil
+      # ]
       deletion_files = [
         "iceberg-position-delete",
         files.map.with_index.select { |v, i| v[:delete_file] }.to_h { |v, i| [i, [v[:delete_file]]] }
       ]
-      Polars.scan_parquet(sources, _deletion_files: deletion_files, storage_options: polars_storage_options)
+      Polars.scan_parquet(
+        sources,
+        storage_options: polars_storage_options,
+        # allow_missing_columns: true,
+        # extra_columns: "ignore",
+        # _column_mapping: column_mapping,
+        _deletion_files: deletion_files
+      )
     end
 
     # libduckdb does not provide function
@@ -366,11 +461,15 @@ module DuckLake
 
     def error_mapping
       @error_mapping ||= {
+        "Binder Error: " => BinderError,
         "Catalog Error: " => CatalogError,
         "Conversion Error: " => ConversionError,
+        "Invalid Configuration Error: " => InvalidConfigurationError,
         "Invalid Input Error: " => InvalidInputError,
         "IO Error: " => IOError,
-        "Permission Error: " => PermissionError
+        "Not implemented Error: " => NotImplementedError,
+        "Permission Error: " => PermissionError,
+        "TransactionContext Error: " => TransactionContextError
       }
     end
 
@@ -409,6 +508,11 @@ module DuckLake
       name
     end
 
+    def set_commit_message(message, author)
+      execute("CALL ducklake_set_commit_message(?, ?, ?)", [@catalog, author, message])
+      nil
+    end
+
     def symbolize_keys(result)
       result.map { |v| v.transform_keys(&:to_sym) }
     end

data/lib/ducklake/version.rb

@@ -1,3 +1,3 @@
 module DuckLake
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

data/lib/ducklake.rb

@@ -11,9 +11,14 @@ require_relative "ducklake/version"
 
 module DuckLake
   class Error < StandardError; end
+  class BinderError < Error; end
   class CatalogError < Error; end
   class ConversionError < Error; end
+  class InvalidConfigurationError < Error; end
   class InvalidInputError < Error; end
   class IOError < Error; end
+  class NotImplementedError < Error; end
   class PermissionError < Error; end
+  class Rollback < Error; end
+  class TransactionContextError < Error; end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ducklake
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Andrew Kane