ducklake 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e31c96f2bcf9728685f6eb72f3cdfad44174eb28347dbe549391703f2795414
-  data.tar.gz: 63befb2ceaad4c0587b6ef10672e687aa2c51cdc78c95451c85599595d9e83d7
+  metadata.gz: 451f6059faf0e7c451599b218bb9dafbbb8b80a2b54416833a5dafb85a0d1a9d
+  data.tar.gz: a1c6c7825da3dd5a077705c1c3f106a0cc1d75bc2e60b17ccb5099f98a3bdf1a
 SHA512:
-  metadata.gz: 1a62d39d7962cbdd8b60a49ba0f7a20d47bbf2a3a18061ae59330cab07b67d91d7dab533487da99cdfbfc3d2b1d7cceaf407bfdd471ffe2bee6d84d2f5567413
-  data.tar.gz: 7032e761c77d93463beb4fb650e17fba8a8e3d54554ca5debc9f14a1625a18806c9167f5719cd7bab7079b068adae151b53a404600ac09b02eb1974cfe90c06e
+  metadata.gz: 9359cf3aaf06ed4398b9bc0ff35489c0e50008cd8d3426a787339d33f541a6bce8e3a1eca4a2b88b46852545895b591d08613f88ad26902aa4d41cf8b842bc93
+  data.tar.gz: 5bd9818d1b928aa2c0a11301c3455b89f8a4fc99912c6d1b8e52fdd151088adcc331b0ca01169df08412e151841f2f6b50f4c7b85b909ca558d6c21f0ffebd56

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.1 (2025-08-18)
+
+- Added experimental support for Polars
+
 ## 0.1.0 (2025-08-17)
 
 - First release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -1,12 +1,12 @@
 # DuckLake Ruby
 
-:fire: [DuckLake](https://ducklake.select/) for Ruby
+:duck: [DuckLake](https://ducklake.select/) for Ruby
 
 Run your own data lake with a SQL database and file/object storage
 
 ```ruby
 DuckLake::Client.new(
-  catalog_url: "postgres://user:pass@host:5432/db",
+  catalog_url: "postgres://user:pass@host:5432/dbname",
   storage_url: "s3://my-bucket/"
 )
 ```
@@ -149,7 +149,7 @@ Or [register existing data files](https://ducklake.select/docs/stable/duckdb/met
 ducklake.add_data_files("events", "data.parquet")
 ```
 
-Note: This transfers ownership to DuckLake, so the file can be deleted after running `cleanup_old_files`
+Note: This transfers ownership to the data lake, so the file may be deleted as part of [maintenance](#maintenance)
 
 Update data
 
@@ -233,13 +233,31 @@ Or for a specific table
 ducklake.set_option("parquet_compression", "zstd", table_name: "events")
 ```
 
-## Security
+## Read-Only Mode
 
-See [best practices](https://duckdb.org/docs/stable/operations_manual/securing_duckdb/overview.html) for DuckDB security.
+Note: This feature is experimental
 
-Grant minimal permissions for the catalog database and data storage.
+Connect to the data lake in read-only mode
 
-### External Access
+```ruby
+DuckLake::Client.new(read_only: true, ...)
+```
+
+Use read-only credentials for catalog database and storage provider and [disable external access](#external-access)
+
+You should also consider [disabling community extensions](https://duckdb.org/docs/stable/operations_manual/securing_duckdb/securing_extensions.html#community-extensions)
+
+```ruby
+ducklake.sql("SET allow_community_extensions = false")
+```
+
+And [locking the configuration](https://duckdb.org/docs/stable/operations_manual/securing_duckdb/overview.html#locking-configurations)
+
+```ruby
+ducklake.sql("SET lock_configuration = true")
+```
+
+## External Access
 
 [Restrict external access](https://duckdb.org/docs/stable/operations_manual/securing_duckdb/overview.html#restricting-file-access) to the DuckDB engine
 
@@ -258,6 +276,40 @@ ducklake.disable_external_access(
 
 The storage URL is automatically included in `allowed_directories`
 
+## SQL Safety
+
+Use parameterized queries when possible
+
+```ruby
+ducklake.sql("SELECT * FROM events WHERE id = ?", [1])
+```
+
+For places that do not support parameters, use `quote` or `quote_identifier`
+
+```ruby
+quoted_table = ducklake.quote_identifier("events")
+quoted_file = ducklake.quote("path/to/data.csv")
+ducklake.sql("COPY #{quoted_table} FROM #{quoted_file}")
+```
+
+## Polars
+
+Note: This feature is experimental and does not work on tables with schema changes
+
+Query the data with [Ruby Polars](https://github.com/ankane/ruby-polars)
+
+```ruby
+ducklake.polars("events")
+```
+
+Specify a snapshot
+
+```ruby
+ducklake.polars("events", snapshot_version: 3)
+# or
+ducklake.polars("events", snapshot_time: Date.today - 7)
+```
+
 ## Reference
 
 Get table info

data/lib/ducklake/client.rb

@@ -8,7 +8,7 @@ module DuckLake
       snapshot_time: nil,
       data_inlining_row_limit: 0,
       create_if_not_exists: false,
-      _read_only: false # experimental
+      read_only: false # experimental
     )
       catalog_uri = URI.parse(catalog_url)
       storage_uri = URI.parse(storage_url)
@@ -30,6 +30,9 @@ module DuckLake
         raise ArgumentError, "Unsupported catalog type: #{catalog_uri.scheme}"
       end
 
+      @storage_scheme = storage_uri.scheme
+      @storage_options = storage_options.dup
+
       secret_options = nil
       storage_options = storage_options.dup
 
@@ -54,7 +57,7 @@ module DuckLake
       end
 
       attach_options = {data_path: storage_url}
-      attach_options[:read_only] = true if _read_only
+      attach_options[:read_only] = true if read_only
       attach_options[:snapshot_version] = snapshot_version if !snapshot_version.nil?
       attach_options[:snapshot_time] = snapshot_time if !snapshot_time.nil?
       attach_options[:data_inlining_row_limit] = data_inlining_row_limit if data_inlining_row_limit > 0
@@ -63,7 +66,7 @@ module DuckLake
       @catalog = "ducklake"
       @storage_url = storage_url
 
-      if _read_only
+      if read_only
         config = DuckDB::Config.new
         config["access_mode"] = "READ_ONLY"
 
@@ -285,6 +288,18 @@ module DuckLake
       nil
     end
 
+    # experimental
+    # TODO support schema changes
+    def polars(table, snapshot_version: nil, snapshot_time: nil)
+      files = list_files(table, snapshot_version:, snapshot_time:)
+      sources = files.map { |v| v[:data_file] }
+      deletion_files = [
+        "iceberg-position-delete",
+        files.map.with_index.select { |v, i| v[:delete_file] }.to_h { |v, i| [i, [v[:delete_file]]] }
+      ]
+      Polars.scan_parquet(sources, _deletion_files: deletion_files, storage_options: polars_storage_options)
+    end
+
     # libduckdb does not provide function
     # https://duckdb.org/docs/stable/sql/dialect/keywords_and_identifiers.html
     def quote_identifier(value)
@@ -412,6 +427,27 @@ module DuckLake
       uri.path[1..]
     end
 
+    def polars_storage_options
+      @polars_storage_options ||= begin
+        storage_options = {}
+        extra_options = @storage_options.dup
+
+        case @storage_scheme
+        when "s3"
+          # https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html
+          [:aws_access_key_id, :aws_secret_access_key, :region].each do |k|
+            storage_options[k] = extra_options.delete(k) if extra_options.key?(k)
+          end
+        end
+
+        if extra_options.any?
+          raise ArgumentError, "Unsupported #{@storage_scheme || "file"} storage options: #{extra_options.keys.map(&:inspect).join(", ")}"
+        end
+
+        storage_options
+      end
+    end
+
     def quote_array(value)
       "[#{value.map { |v| quote(v) }.join(", ")}]"
     end

data/lib/ducklake/version.rb

@@ -1,3 +1,3 @@
 module DuckLake
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ducklake
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Andrew Kane
@@ -29,6 +29,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
+- LICENSE.txt
 - README.md
 - lib/ducklake.rb
 - lib/ducklake/client.rb