ducklake 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e31c96f2bcf9728685f6eb72f3cdfad44174eb28347dbe549391703f2795414
-  data.tar.gz: 63befb2ceaad4c0587b6ef10672e687aa2c51cdc78c95451c85599595d9e83d7
+  metadata.gz: 9366806f0dee7b0277dfee65be6c4718c339b6e9ab53e595e1f88534d5144534
+  data.tar.gz: 3b3c01d575bb8d0c380b49cbbc8b6deb62cc7de5542449dc73d0fd3b5ccffb1f
 SHA512:
-  metadata.gz: 1a62d39d7962cbdd8b60a49ba0f7a20d47bbf2a3a18061ae59330cab07b67d91d7dab533487da99cdfbfc3d2b1d7cceaf407bfdd471ffe2bee6d84d2f5567413
-  data.tar.gz: 7032e761c77d93463beb4fb650e17fba8a8e3d54554ca5debc9f14a1625a18806c9167f5719cd7bab7079b068adae151b53a404600ac09b02eb1974cfe90c06e
+  metadata.gz: 2e1fb28d47b8efeedaedba54727d678bdf946efd6aac1001ae0a2ad9aa9fbdc5b28bf8df06b2a10be13f6684195e2f684eafe71c7bedb27b36eff7af36d65abf
+  data.tar.gz: 4feea1505445b4347e722ea31d87c6dd9ed759489550f7a69997fced7b9a45827b4beaa81c4bc5e12abb801226af82f1c4903aff5ab6fd4b0851b6772b2c83c3

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.1.2 (2025-08-23)
+
+- Added `transaction` method
+
+## 0.1.1 (2025-08-18)
+
+- Added experimental support for Polars
+
 ## 0.1.0 (2025-08-17)
 
 - First release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -1,12 +1,12 @@
 # DuckLake Ruby
 
-:fire: [DuckLake](https://ducklake.select/) for Ruby
+:duck: [DuckLake](https://ducklake.select/) for Ruby
 
 Run your own data lake with a SQL database and file/object storage
 
 ```ruby
 DuckLake::Client.new(
-  catalog_url: "postgres://user:pass@host:5432/db",
+  catalog_url: "postgres://user:pass@host:5432/dbname",
   storage_url: "s3://my-bucket/"
 )
 ```
@@ -149,7 +149,7 @@ Or [register existing data files](https://ducklake.select/docs/stable/duckdb/met
 ducklake.add_data_files("events", "data.parquet")
 ```
 
-Note: This transfers ownership to DuckLake, so the file can be deleted after running `cleanup_old_files`
+Note: This transfers ownership to the data lake, so the file may be deleted as part of [maintenance](#maintenance)
 
 Update data
 
@@ -163,12 +163,46 @@ Delete data
 ducklake.sql("DELETE * FROM events WHERE id = ?", [1])
 ```
 
+Run multiple statements in a transaction
+
+```ruby
+ducklake.transaction do
+  # ...
+end
+```
+
+Raise `DuckLake::Rollback` to rollback
+
+## Schema Changes
+
 Update the schema
 
 ```ruby
 ducklake.sql("ALTER TABLE events ADD COLUMN active BOOLEAN")
 ```
 
+Set or remove a [partitioning key](https://ducklake.select/docs/stable/duckdb/advanced_features/partitioning)
+
+```ruby
+ducklake.sql("ALTER TABLE events SET PARTITIONED BY (name)")
+# or
+ducklake.sql("ALTER TABLE events RESET PARTITIONED BY")
+```
+
+## Views
+
+Create a view
+
+```ruby
+ducklake.sql("CREATE VIEW events_view AS SELECT * FROM events")
+```
+
+Drop a view
+
+```ruby
+ducklake.sql("DROP VIEW events_view")
+```
+
 ## Snapshots
 
 Get snapshots
@@ -233,13 +267,31 @@ Or for a specific table
 ducklake.set_option("parquet_compression", "zstd", table_name: "events")
 ```
 
-## Security
+## Read-Only Mode
+
+Note: This feature is experimental and does not prevent the DuckDB engine from writing files via `sql`
+
+Attach the catalog in read-only mode
+
+```ruby
+DuckLake::Client.new(read_only: true, ...)
+```
+
+Use read-only credentials for catalog database and storage provider and [disable external access](#external-access)
 
-See [best practices](https://duckdb.org/docs/stable/operations_manual/securing_duckdb/overview.html) for DuckDB security.
+You should also consider [disabling community extensions](https://duckdb.org/docs/stable/operations_manual/securing_duckdb/securing_extensions.html#community-extensions)
 
-Grant minimal permissions for the catalog database and data storage.
+```ruby
+ducklake.sql("SET allow_community_extensions = false")
+```
 
-### External Access
+And [locking the configuration](https://duckdb.org/docs/stable/operations_manual/securing_duckdb/overview.html#locking-configurations)
+
+```ruby
+ducklake.sql("SET lock_configuration = true")
+```
+
+## External Access
 
 [Restrict external access](https://duckdb.org/docs/stable/operations_manual/securing_duckdb/overview.html#restricting-file-access) to the DuckDB engine
 
@@ -258,6 +310,40 @@ ducklake.disable_external_access(
 
 The storage URL is automatically included in `allowed_directories`
 
+## SQL Safety
+
+Use parameterized queries when possible
+
+```ruby
+ducklake.sql("SELECT * FROM events WHERE id = ?", [1])
+```
+
+For places that do not support parameters, use `quote` or `quote_identifier`
+
+```ruby
+quoted_table = ducklake.quote_identifier("events")
+quoted_file = ducklake.quote("path/to/data.csv")
+ducklake.sql("COPY #{quoted_table} FROM #{quoted_file}")
+```
+
+## Polars
+
+Note: This feature is experimental and does not currently work on tables with schema changes
+
+Query the data with [Ruby Polars](https://github.com/ankane/ruby-polars)
+
+```ruby
+ducklake.polars("events")
+```
+
+Specify a snapshot
+
+```ruby
+ducklake.polars("events", snapshot_version: 3)
+# or
+ducklake.polars("events", snapshot_time: Date.today - 7)
+```
+
 ## Reference
 
 Get table info

data/lib/ducklake/client.rb

@@ -8,7 +8,7 @@ module DuckLake
       snapshot_time: nil,
       data_inlining_row_limit: 0,
       create_if_not_exists: false,
-      _read_only: false # experimental
+      read_only: false # experimental
     )
       catalog_uri = URI.parse(catalog_url)
       storage_uri = URI.parse(storage_url)
@@ -30,6 +30,9 @@ module DuckLake
         raise ArgumentError, "Unsupported catalog type: #{catalog_uri.scheme}"
       end
 
+      @storage_scheme = storage_uri.scheme
+      @storage_options = storage_options.dup
+
       secret_options = nil
       storage_options = storage_options.dup
 
@@ -54,7 +57,7 @@ module DuckLake
       end
 
       attach_options = {data_path: storage_url}
-      attach_options[:read_only] = true if _read_only
+      attach_options[:read_only] = true if read_only
       attach_options[:snapshot_version] = snapshot_version if !snapshot_version.nil?
       attach_options[:snapshot_time] = snapshot_time if !snapshot_time.nil?
       attach_options[:data_inlining_row_limit] = data_inlining_row_limit if data_inlining_row_limit > 0
@@ -63,7 +66,7 @@ module DuckLake
       @catalog = "ducklake"
       @storage_url = storage_url
 
-      if _read_only
+      if read_only
         config = DuckDB::Config.new
         config["access_mode"] = "READ_ONLY"
 
@@ -103,6 +106,17 @@ module DuckLake
       execute(sql, params)
     end
 
+    def transaction
+      execute("BEGIN")
+      begin
+        yield
+        execute("COMMIT")
+      rescue => e
+        execute("ROLLBACK")
+        raise e unless e.is_a?(Rollback)
+      end
+    end
+
     def attach(alias_, url)
       type = nil
       extension = nil
@@ -148,6 +162,15 @@ module DuckLake
       symbolize_keys result
     end
 
+    # experimental
+    # TODO use keyword arguments or range?
+    def table_changes(table, start_snapshot, end_snapshot)
+      params = [@catalog, "main", table, start_snapshot, end_snapshot]
+      result = execute("SELECT * FROM ducklake_table_changes(?, ?, ?, ?, ?)", params)
+      # only return changes between snapshots
+      symbolize_keys result.reject { |v| v["snapshot_id"] == start_snapshot }
+    end
+
     # TODO more DDL methods?
     def drop_table(table, if_exists: nil)
       execute("DROP TABLE#{" IF EXISTS" if if_exists} #{quote_identifier(table)}")
@@ -285,6 +308,29 @@ module DuckLake
       nil
     end
 
+    # experimental
+    def polars(table, snapshot_version: nil, snapshot_time: nil)
+      files = list_files(table, snapshot_version:, snapshot_time:)
+      sources = files.map { |v| v[:data_file] }
+      # TODO support schema changes
+      # column_mapping = [
+      #   "iceberg-column-mapping",
+      #   nil
+      # ]
+      deletion_files = [
+        "iceberg-position-delete",
+        files.map.with_index.select { |v, i| v[:delete_file] }.to_h { |v, i| [i, [v[:delete_file]]] }
+      ]
+      Polars.scan_parquet(
+        sources,
+        storage_options: polars_storage_options,
+        # allow_missing_columns: true,
+        # extra_columns: "ignore",
+        # _column_mapping: column_mapping,
+        _deletion_files: deletion_files
+      )
+    end
+
     # libduckdb does not provide function
     # https://duckdb.org/docs/stable/sql/dialect/keywords_and_identifiers.html
     def quote_identifier(value)
@@ -355,7 +401,9 @@ module DuckLake
         "Conversion Error: " => ConversionError,
         "Invalid Input Error: " => InvalidInputError,
         "IO Error: " => IOError,
-        "Permission Error: " => PermissionError
+        "Not implemented Error: " => NotImplementedError,
+        "Permission Error: " => PermissionError,
+        "TransactionContext Error: " => TransactionContextError
       }
     end
 
@@ -412,6 +460,27 @@ module DuckLake
       uri.path[1..]
     end
 
+    def polars_storage_options
+      @polars_storage_options ||= begin
+        storage_options = {}
+        extra_options = @storage_options.dup
+
+        case @storage_scheme
+        when "s3"
+          # https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html
+          [:aws_access_key_id, :aws_secret_access_key, :region].each do |k|
+            storage_options[k] = extra_options.delete(k) if extra_options.key?(k)
+          end
+        end
+
+        if extra_options.any?
+          raise ArgumentError, "Unsupported #{@storage_scheme || "file"} storage options: #{extra_options.keys.map(&:inspect).join(", ")}"
+        end
+
+        storage_options
+      end
+    end
+
     def quote_array(value)
       "[#{value.map { |v| quote(v) }.join(", ")}]"
     end

data/lib/ducklake/version.rb

@@ -1,3 +1,3 @@
 module DuckLake
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/ducklake.rb

@@ -15,5 +15,8 @@ module DuckLake
   class ConversionError < Error; end
   class InvalidInputError < Error; end
   class IOError < Error; end
+  class NotImplementedError < Error; end
   class PermissionError < Error; end
+  class Rollback < Error; end
+  class TransactionContextError < Error; end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ducklake
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Kane
@@ -29,6 +29,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
+- LICENSE.txt
 - README.md
 - lib/ducklake.rb
 - lib/ducklake/client.rb