seaduck 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7b1b2081ef2387c773e8d54e81f82b02573efdc8f6be7fb45083c359cba6ce18
+  data.tar.gz: 915186c80411757817991858b892f6dba31c353339a8eea6317299f8a1389683
+SHA512:
+  metadata.gz: df40e3b4113ac02773e22db0bd66afe3b97ab93cf9c2bd2dedec4ef5390ae70931a09be48291d07bd263ea336839b04307f7c32ce46a77e9676d4c6ffab2ebff
+  data.tar.gz: 87a754ad9eaedc3cec9a71c7c02bca8544a96ea6767ae601c3fa7efc0e4ac850771f2220c1164bfeaaf6447fcbdcc6254718e02f3436308f109d6a313983d37b

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0 (2025-09-30)
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,163 @@
+# SeaDuck
+
+[Apache Iceberg](https://iceberg.apache.org/) for Ruby, powered by libduckdb
+
+[![Build Status](https://github.com/ankane/seaduck/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/seaduck/actions)
+
+## Installation
+
+First, install libduckdb. For Homebrew, use:
+
+```sh
+brew install duckdb
+```
+
+Then add this line to your application’s Gemfile:
+
+```ruby
+gem "seaduck"
+```
+
+## Getting Started
+
+Create a client for an Iceberg catalog
+
+```ruby
+catalog = SeaDuck::S3TablesCatalog.new(arn: "arn:aws:s3tables:...")
+```
+
+Note: SeaDuck requires a default namespace, which is `main` by default. This namespace is created if it does not exist. Pass `default_namespace` to use a different one.
+
+Create a table
+
+```ruby
+catalog.sql("CREATE TABLE events (id bigint, name text)")
+```
+
+Load data from a file
+
+```ruby
+catalog.sql("COPY events FROM 'data.csv'")
+```
+
+You can also load data directly from other [data sources](https://duckdb.org/docs/stable/data/data_sources)
+
+```ruby
+catalog.attach("blog", "postgres://localhost:5432/blog")
+catalog.sql("INSERT INTO events SELECT * FROM blog.ahoy_events")
+```
+
+Query the data
+
+```ruby
+catalog.sql("SELECT COUNT(*) FROM events").to_a
+```
+
+## Namespaces
+
+List namespaces
+
+```ruby
+catalog.list_namespaces
+```
+
+Create a namespace
+
+```ruby
+catalog.create_namespace("main")
+```
+
+Check if a namespace exists
+
+```ruby
+catalog.namespace_exists?("main")
+```
+
+Drop a namespace
+
+```ruby
+catalog.drop_namespace("main")
+```
+
+## Tables
+
+List tables
+
+```ruby
+catalog.list_tables
+```
+
+Check if a table exists
+
+```ruby
+catalog.table_exists?("events")
+```
+
+Drop a table
+
+```ruby
+catalog.drop_table("events")
+```
+
+## Snapshots
+
+Get snapshots for a table
+
+```ruby
+catalog.snapshots("events")
+```
+
+Query the data at a specific snapshot version or time
+
+```ruby
+catalog.sql("SELECT * FROM events AT (VERSION => ?)", [3])
+# or
+catalog.sql("SELECT * FROM events AT (TIMESTAMP => ?)", [Date.today - 7])
+```
+
+## SQL Safety
+
+Use parameterized queries when possible
+
+```ruby
+catalog.sql("SELECT * FROM events WHERE id = ?", [1])
+```
+
+For places that do not support parameters, use `quote` or `quote_identifier`
+
+```ruby
+quoted_table = catalog.quote_identifier("events")
+quoted_file = catalog.quote("path/to/data.csv")
+catalog.sql("COPY #{quoted_table} FROM #{quoted_file}")
+```
+
+## History
+
+View the [changelog](https://github.com/ankane/seaduck/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/seaduck/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/seaduck/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone https://github.com/ankane/seaduck.git
+cd seaduck
+bundle install
+
+# REST catalog
+docker compose up
+bundle exec rake test:rest
+
+# S3 Tables catalog
+bundle exec rake test:s3tables
+
+# Glue catalog
+bundle exec rake test:glue
+```

data/lib/seaduck/catalog.rb

@@ -0,0 +1,255 @@
+module SeaDuck
+  class Catalog
+    def _initialize(url, default_namespace:, attach_options:, secret_options: nil, extensions: [])
+      @catalog = "iceberg"
+      @default_namespace = default_namespace
+
+      @db = DuckDB::Database.open
+      @conn = @db.connect
+
+      install_extension("iceberg")
+      extensions.each do |extension|
+        install_extension(extension)
+      end
+      create_secret(secret_options) if secret_options
+      attach_with_options(@catalog, url, {type: "iceberg"}.merge(attach_options))
+
+      begin
+        use_namespace(@default_namespace)
+      rescue Error
+        create_namespace(@default_namespace, if_not_exists: true)
+        use_namespace(@default_namespace)
+      end
+      execute("DETACH memory")
+    end
+
+    def list_namespaces
+      execute("SELECT schema_name FROM information_schema.schemata WHERE catalog_name = ?", [@catalog]).rows
+    end
+
+    def create_namespace(namespace, if_not_exists: nil)
+      execute("CREATE SCHEMA#{" IF NOT EXISTS" if if_not_exists} #{quote_namespace(namespace)}")
+      nil
+    end
+
+    def namespace_exists?(namespace)
+      execute("SELECT 1 FROM information_schema.schemata WHERE catalog_name = ? AND schema_name = ?", [@catalog, namespace]).any?
+    end
+
+    # CASCADE not implemented for Iceberg yet
+    def drop_namespace(namespace, if_exists: nil)
+      execute("DROP SCHEMA#{" IF EXISTS" if if_exists} #{quote_namespace(namespace)}")
+      nil
+    end
+
+    def list_tables(namespace = nil)
+      sql = +"SELECT table_schema, table_name FROM information_schema.tables WHERE table_catalog = ?"
+      params = [@catalog]
+
+      if namespace
+        sql << " AND table_schema = ?"
+        params << namespace
+      end
+
+      execute(sql, params).rows
+    end
+
+    def table_exists?(table_name)
+      namespace, table_name = split_table(table_name)
+      execute("SELECT 1 FROM information_schema.tables WHERE table_catalog = ? AND table_schema = ? AND table_name = ?", [@catalog, namespace, table_name]).any?
+    end
+
+    def drop_table(table_name, if_exists: nil)
+      execute("DROP TABLE#{" IF EXISTS" if if_exists} #{quote_table(table_name)}")
+    end
+
+    def snapshots(table_name)
+      symbolize_keys execute("SELECT * FROM iceberg_snapshots(#{quote_table(table_name)})")
+    end
+
+    def sql(sql, params = [])
+      execute(sql, params)
+    end
+
+    def transaction
+      execute("BEGIN")
+      begin
+        yield
+        execute("COMMIT")
+      rescue => e
+        execute("ROLLBACK")
+        raise e unless e.is_a?(Rollback)
+      end
+    end
+
+    def attach(alias_, url)
+      type = nil
+      extension = nil
+
+      uri = URI.parse(url)
+      case uri.scheme
+      when "postgres", "postgresql"
+        type = "postgres"
+        extension = "postgres"
+      else
+        raise ArgumentError, "Unsupported data source type: #{uri.scheme}"
+      end
+
+      install_extension(extension) if extension
+
+      options = {
+        type: type,
+        read_only: true
+      }
+      attach_with_options(alias_, url, options)
+    end
+
+    def detach(alias_)
+      execute("DETACH #{quote_identifier(alias_)}")
+      nil
+    end
+
+    # libduckdb does not provide function
+    # https://duckdb.org/docs/stable/sql/dialect/keywords_and_identifiers.html
+    def quote_identifier(value)
+      "\"#{encoded(value).gsub('"', '""')}\""
+    end
+
+    # libduckdb does not provide function
+    # TODO support more types
+    def quote(value)
+      if value.nil?
+        "NULL"
+      elsif value == true
+        "true"
+      elsif value == false
+        "false"
+      elsif defined?(BigDecimal) && value.is_a?(BigDecimal)
+        value.to_s("F")
+      elsif value.is_a?(Numeric)
+        value.to_s
+      else
+        if value.is_a?(Time)
+          value = value.utc.iso8601(9)
+        elsif value.is_a?(DateTime)
+          value = value.iso8601(9)
+        elsif value.is_a?(Date)
+          value = value.strftime("%Y-%m-%d")
+        end
+        "'#{encoded(value).gsub("'", "''")}'"
+      end
+    end
+
+    # hide internal state
+    def inspect
+      to_s
+    end
+
+    private
+
+    def execute(sql, params = [])
+      # use prepare instead of query to prevent multiple statements at once
+      result =
+        @conn.prepare(sql) do |stmt|
+          params.each_with_index do |v, i|
+            stmt.bind(i + 1, v)
+          end
+          stmt.execute
+        end
+
+      # TODO add column types
+      Result.new(result.columns.map(&:name), result.to_a)
+    rescue DuckDB::Error => e
+      raise map_error(e), cause: nil
+    end
+
+    def error_mapping
+      @error_mapping ||= {
+        "Binder Error: " => BinderError,
+        "Catalog Error: " => CatalogError,
+        "Conversion Error: " => ConversionError,
+        "Invalid Configuration Error: " => InvalidConfigurationError,
+        "Invalid Input Error: " => InvalidInputError,
+        "IO Error: " => IOError,
+        "Not implemented Error: " => NotImplementedError,
+        "Permission Error: " => PermissionError,
+        "TransactionContext Error: " => TransactionContextError
+      }
+    end
+
+    # not ideal to base on prefix, but do not see a better way at the moment
+    def map_error(e)
+      error_mapping.each do |prefix, cls|
+        if e.message&.start_with?(prefix)
+          return cls.new(e.message.delete_prefix(prefix))
+        end
+      end
+      Error.new(e.message)
+    end
+
+    def install_extension(extension)
+      execute("INSTALL #{quote_identifier(extension)}")
+    end
+
+    def create_secret(options)
+      execute("CREATE SECRET (#{options_args(options)})")
+    end
+
+    def attach_with_options(alias_, url, options)
+      execute("ATTACH #{quote(url)} AS #{quote_identifier(alias_)} (#{options_args(options)})")
+    end
+
+    def options_args(options)
+      options.map { |k, v| "#{option_name(k)} #{quote(v)}" }.join(", ")
+    end
+
+    def option_name(k)
+      name = k.to_s.upcase
+      # should never contain user input, but just to be safe
+      unless name.match?(/\A[A-Z_]+\z/)
+        raise "Invalid option name"
+      end
+      name
+    end
+
+    def symbolize_keys(result)
+      result.map { |v| v.transform_keys(&:to_sym) }
+    end
+
+    def use_namespace(namespace)
+      execute("USE #{quote_namespace(namespace)}")
+    end
+
+    def quote_namespace(value)
+      "#{quote_identifier(@catalog)}.#{quote_identifier(value)}"
+    end
+
+    def split_table(value)
+      if value.is_a?(Array)
+        if value.size == 2
+          value
+        else
+          raise ArgumentError, "Invalid table identifier"
+        end
+      else
+        [@default_namespace, value]
+      end
+    end
+
+    def quote_table(value)
+      namespace, table_name = split_table(value)
+      "#{quote_namespace(namespace)}.#{quote_identifier(table_name)}"
+    end
+
+    def encoded(value)
+      value = value.to_s if value.is_a?(Symbol)
+      if !value.respond_to?(:to_str)
+        raise TypeError, "no implicit conversion of #{value.class.name} into String"
+      end
+      if ![Encoding::UTF_8, Encoding::US_ASCII].include?(value.encoding) || !value.valid_encoding?
+        raise ArgumentError, "Unsupported encoding"
+      end
+      value
+    end
+  end
+end

data/lib/seaduck/glue_catalog.rb

@@ -0,0 +1,20 @@
+module SeaDuck
+  class GlueCatalog < Catalog
+    # https://duckdb.org/docs/stable/core_extensions/iceberg/amazon_sagemaker_lakehouse
+    def initialize(warehouse:, default_namespace: "main")
+      attach_options = {
+        endpoint_type: "glue"
+      }
+      secret_options = {
+        type: "s3",
+        provider: "credential_chain"
+      }
+      _initialize(
+        warehouse,
+        default_namespace:,
+        attach_options:,
+        secret_options:
+      )
+    end
+  end
+end

data/lib/seaduck/rest_catalog.rb

@@ -0,0 +1,16 @@
+module SeaDuck
+  class RestCatalog < Catalog
+    def initialize(uri:, warehouse: nil, default_namespace: "main", _secret_options: nil)
+      attach_options = {
+        endpoint: uri,
+        authorization_type: "none"
+      }
+      _initialize(
+        warehouse.to_s,
+        default_namespace:,
+        attach_options:,
+        secret_options: _secret_options
+      )
+    end
+  end
+end

data/lib/seaduck/result.rb

@@ -0,0 +1,22 @@
+module SeaDuck
+  class Result
+    include Enumerable
+
+    attr_reader :columns, :rows
+
+    def initialize(columns, rows)
+      @columns = columns
+      @rows = rows
+    end
+
+    def each
+      @rows.each do |row|
+        yield @columns.zip(row).to_h
+      end
+    end
+
+    def empty?
+      rows.empty?
+    end
+  end
+end

data/lib/seaduck/s3_tables_catalog.rb

@@ -0,0 +1,21 @@
+module SeaDuck
+  class S3TablesCatalog < Catalog
+    # https://duckdb.org/docs/stable/core_extensions/iceberg/amazon_s3_tables
+    def initialize(arn:, default_namespace: "main")
+      attach_options = {
+        endpoint_type: "s3_tables"
+      }
+      secret_options = {
+        type: "s3",
+        provider: "credential_chain"
+      }
+      _initialize(
+        arn,
+        default_namespace:,
+        attach_options:,
+        secret_options:,
+        extensions: ["aws", "httpfs"]
+      )
+    end
+  end
+end

data/lib/seaduck/version.rb

@@ -0,0 +1,3 @@
+module SeaDuck
+  VERSION = "0.1.0"
+end

data/lib/seaduck.rb

@@ -0,0 +1,29 @@
+# dependencies
+require "duckdb"
+
+# stdlib
+require "uri"
+
+# modules
+require_relative "seaduck/catalog"
+require_relative "seaduck/result"
+require_relative "seaduck/version"
+
+# catalogs
+require_relative "seaduck/glue_catalog"
+require_relative "seaduck/rest_catalog"
+require_relative "seaduck/s3_tables_catalog"
+
+module SeaDuck
+  class Error < StandardError; end
+  class BinderError < Error; end
+  class CatalogError < Error; end
+  class ConversionError < Error; end
+  class InvalidConfigurationError < Error; end
+  class InvalidInputError < Error; end
+  class IOError < Error; end
+  class NotImplementedError < Error; end
+  class PermissionError < Error; end
+  class Rollback < Error; end
+  class TransactionContextError < Error; end
+end

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification
+name: seaduck
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: duckdb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+email: andrew@ankane.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/seaduck.rb
+- lib/seaduck/catalog.rb
+- lib/seaduck/glue_catalog.rb
+- lib/seaduck/rest_catalog.rb
+- lib/seaduck/result.rb
+- lib/seaduck/s3_tables_catalog.rb
+- lib/seaduck/version.rb
+homepage: https://github.com/ankane/seaduck
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Apache Iceberg for Ruby, powered by libduckdb
+test_files: []