iceberg 0.7.0 → 0.10.1

data/lib/iceberg/catalog.rb

@@ -0,0 +1,107 @@
+module Iceberg
+  class Catalog
+    def list_namespaces(parent = nil)
+      @catalog.list_namespaces(parent)
+    end
+
+    def create_namespace(namespace, properties: {}, if_not_exists: nil)
+      @catalog.create_namespace(namespace, properties)
+    rescue Error => e
+      if !if_not_exists || (e.message != "Cannot create namespace" && !e.message.include?("already exists"))
+        raise e
+      end
+      nil
+    end
+
+    def namespace_exists?(namespace)
+      @catalog.namespace_exists?(namespace)
+    end
+
+    def namespace_properties(namespace)
+      @catalog.namespace_properties(namespace)
+    end
+
+    def update_namespace(namespace, properties:)
+      @catalog.update_namespace(namespace, properties)
+    end
+
+    def drop_namespace(namespace, if_exists: nil)
+      @catalog.drop_namespace(namespace)
+    rescue Error => e
+      if !if_exists || (e.message != "Tried to drop a namespace that does not exist" && !e.message.include?("No such namespace"))
+        raise e
+      end
+      nil
+    end
+
+    def list_tables(namespace)
+      @catalog.list_tables(namespace)
+    end
+
+    def create_table(table_name, schema: nil, location: nil)
+      if !schema.nil? && block_given?
+        raise ArgumentError, "Must pass schema or block"
+      end
+
+      if block_given?
+        table_definition = TableDefinition.new
+        yield table_definition
+        schema = Schema.new(table_definition.fields)
+      elsif schema.is_a?(Hash)
+        fields =
+          schema.map.with_index do |(k, v), i|
+            {
+              id: i + 1,
+              name: k.is_a?(Symbol) ? k.to_s : k,
+              type: v,
+              required: false
+            }
+          end
+        schema = Schema.new(fields)
+      elsif schema.nil?
+        schema = Schema.new([])
+      end
+
+      Table.new(@catalog.create_table(table_name, schema, location), @catalog)
+    end
+
+    def load_table(table_name)
+      Table.new(@catalog.load_table(table_name), @catalog)
+    end
+
+    def drop_table(table_name, if_exists: nil)
+      @catalog.drop_table(table_name)
+    rescue Error => e
+      if !if_exists || (e.message != "Tried to drop a table that does not exist" && !e.message.include?("No such table"))
+        raise e
+      end
+      nil
+    end
+
+    def table_exists?(table_name)
+      @catalog.table_exists?(table_name)
+    rescue NamespaceNotFoundError
+      false
+    end
+
+    def rename_table(table_name, new_name)
+      @catalog.rename_table(table_name, new_name)
+    end
+
+    def register_table(table_name, metadata_location)
+      @catalog.register_table(table_name, metadata_location)
+    end
+
+    def query(sql)
+      # requires datafusion feature
+      raise Todo unless @catalog.respond_to?(:query)
+
+      @catalog.query(sql)
+    end
+
+    # hide internal state
+    def inspect
+      to_s
+    end
+  end
+end

data/lib/iceberg/glue_catalog.rb

@@ -0,0 +1,11 @@
+module Iceberg
+  class GlueCatalog < Catalog
+    # warehouse is URI of S3 storage bucket
+    def initialize(warehouse:)
+      # requires glue feature
+      raise Error, "Feature not enabled" unless RbCatalog.respond_to?(:new_glue)
+
+      @catalog = RbCatalog.new_glue(warehouse)
+    end
+  end
+end

data/lib/iceberg/memory_catalog.rb

@@ -0,0 +1,8 @@
+module Iceberg
+  class MemoryCatalog < Catalog
+    # warehouse is default storage location
+    def initialize(warehouse: nil)
+      @catalog = RbCatalog.new_memory(warehouse)
+    end
+  end
+end

data/lib/iceberg/rest_catalog.rb

@@ -0,0 +1,8 @@
+module Iceberg
+  class RestCatalog < Catalog
+    # warehouse is passed to REST server
+    def initialize(uri:, warehouse: nil, properties: {})
+      @catalog = RbCatalog.new_rest(uri, warehouse, properties)
+    end
+  end
+end

data/lib/iceberg/schema.rb

@@ -0,0 +1,10 @@
+module Iceberg
+  class Schema
+    attr_reader :fields, :schema_id
+
+    def initialize(fields, schema_id: nil)
+      @fields = fields
+      @schema_id = schema_id
+    end
+  end
+end

data/lib/iceberg/sql_catalog.rb

@@ -0,0 +1,9 @@
+module Iceberg
+  class SqlCatalog < Catalog
+    # warehouse is default storage location
+    # name is stored in SQL table
+    def initialize(uri:, warehouse:, name: "main", properties: {})
+      @catalog = RbCatalog.new_sql(uri, warehouse, name, properties)
+    end
+  end
+end

data/lib/iceberg/static_table.rb

@@ -0,0 +1,8 @@
+module Iceberg
+  class StaticTable < Table
+    def initialize(metadata_location)
+      table = RbTable.from_metadata_file(metadata_location)
+      super(table, nil)
+    end
+  end
+end

data/lib/iceberg/table.rb

@@ -0,0 +1,154 @@
+module Iceberg
+  class Table
+    def initialize(table, catalog)
+      @table = table
+      @catalog = catalog
+    end
+
+    def format_version
+      @table.format_version
+    end
+
+    def uuid
+      @table.uuid
+    end
+
+    def location
+      @table.location
+    end
+
+    def last_sequence_number
+      @table.last_sequence_number
+    end
+
+    def next_sequence_number
+      @table.next_sequence_number
+    end
+
+    def last_column_id
+      @table.last_column_id
+    end
+
+    def last_partition_id
+      @table.last_partition_id
+    end
+
+    def schemas
+      @table.schemas
+    end
+
+    def schema_by_id(schema_id)
+      @table.schema_by_id(schema_id)
+    end
+
+    def current_schema
+      @table.current_schema
+    end
+    alias_method :schema, :current_schema
+
+    def current_schema_id
+      @table.current_schema_id
+    end
+    alias_method :schema_id, :current_schema_id
+
+    def default_partition_spec_id
+      @table.default_partition_spec_id
+    end
+
+    def snapshots
+      @table.snapshots
+    end
+
+    def snapshot_by_id(snapshot_id)
+      @table.snapshot_by_id(snapshot_id)
+    end
+
+    def history
+      @table.history
+    end
+
+    def metadata_log
+      @table.metadata_log
+    end
+
+    def current_snapshot
+      @table.current_snapshot
+    end
+
+    def current_snapshot_id
+      @table.current_snapshot_id
+    end
+
+    def properties
+      @table.properties
+    end
+
+    def to_polars(snapshot_id: nil, storage_options: nil)
+      require "polars-df"
+
+      files = @table.scan(snapshot_id).plan_files
+      if files.empty?
+        # TODO improve
+        schema =
+          current_schema.fields.to_h do |field|
+            dtype =
+              case field[:type]
+              when "int"
+                Polars::Int32
+              when "long"
+                Polars::Int64
+              when "double"
+                Polars::Float64
+              when "string"
+                Polars::String
+              when "timestamp"
+                Polars::Datetime
+              else
+                raise Todo
+              end
+
+            [field[:name], dtype]
+          end
+
+        Polars::LazyFrame.new(schema: schema)
+      else
+        sources = files.map { |v| v[:data_file_path] }
+
+        deletion_files = [
+          "iceberg-position-delete",
+          files.map.with_index
+            .select { |v, i| v[:deletes].any? }
+            .to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
+        ]
+
+        Polars.scan_parquet(
+          sources,
+          storage_options: storage_options,
+          # TODO
+          # cast_options: Polars::ScanCastOptions._default_iceberg,
+          # allow_missing_columns: true,
+          # extra_columns: "ignore",
+          # _column_mapping: column_mapping,
+          _deletion_files: deletion_files
+        )
+      end
+    end
+
+    def append(df)
+      check_catalog
+      @table = @table.append(df.arrow_c_stream, @catalog)
+      nil
+    end
+
+    # hide internal state
+    def inspect
+      to_s
+    end
+
+    private
+
+    def check_catalog
+      raise Error, "Read-only table" if @catalog.nil?
+    end
+  end
+end

data/lib/iceberg/table_definition.rb

@@ -0,0 +1,37 @@
+module Iceberg
+  class TableDefinition
+    TYPES = %w[
+      boolean int long float double date timestamp timestamptz string uuid binary
+    ]
+
+    TYPE_ALIASES = {
+      "integer" => "int",
+      "bigint" => "long"
+    }
+
+    attr_reader :fields
+
+    def initialize
+      @fields = []
+    end
+
+    (TYPES + TYPE_ALIASES.keys).each do |type|
+      define_method type do |name, **options|
+        column(name, type, **options)
+      end
+    end
+
+    def column(name, type, null: true, default: nil, comment: nil)
+      type = type.to_s
+      @fields << {
+        id: @fields.size + 1,
+        name: name.to_s,
+        type: TYPE_ALIASES.fetch(type, type),
+        required: !null,
+        doc: comment,
+        # no need for initial default (and not supported until v3)
+        write_default: default
+      }
+    end
+  end
+end

data/lib/iceberg/version.rb

@@ -0,0 +1,3 @@
+module Iceberg
+  VERSION = "0.10.1"
+end

data/lib/iceberg.rb

@@ -0,0 +1,36 @@
+# ext
+begin
+  require "iceberg/#{RUBY_VERSION.to_f}/iceberg"
+rescue LoadError
+  require "iceberg/iceberg"
+end
+
+# modules
+require_relative "iceberg/catalog"
+require_relative "iceberg/schema"
+require_relative "iceberg/table"
+require_relative "iceberg/static_table"
+require_relative "iceberg/table_definition"
+require_relative "iceberg/version"
+
+# catalogs
+require_relative "iceberg/glue_catalog"
+require_relative "iceberg/memory_catalog"
+require_relative "iceberg/rest_catalog"
+require_relative "iceberg/sql_catalog"
+
+module Iceberg
+  class Error < StandardError; end
+  class InvalidDataError < Error; end
+  class NamespaceAlreadyExistsError < Error; end
+  class NamespaceNotFoundError < Error; end
+  class TableAlreadyExistsError < Error; end
+  class TableNotFoundError < Error; end
+  class UnsupportedFeatureError < Error; end
+
+  class Todo < Error
+    def message
+      "not implemented yet"
+    end
+  end
+end

metadata

@@ -1,136 +1,80 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: iceberg
-version: !ruby/object:Gem::Version 
-  hash: 3
-  prerelease: 
-  segments: 
-  - 0
-  - 7
-  - 0
-  version: 0.7.0
+version: !ruby/object:Gem::Version
+  version: 0.10.1
 platform: ruby
-authors: 
-- stellard
-autorequire: 
+authors:
+- Andrew Kane
 bindir: bin
 cert_chain: []
-
-date: 2011-08-03 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id001
-  type: :development
-  name: shoulda
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
-        version: 1.0.0
-  version_requirements: *id002
-  type: :development
-  name: bundler
-  prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 1
-        - 5
-        - 2
-        version: 1.5.2
-  version_requirements: *id003
-  type: :development
-  name: jeweler
-  prerelease: false
-- !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id004
-  type: :development
-  name: rcov
-  prerelease: false
-description: just a placeholder for now until public release
-email: scott.ellard@gmail.com
+      - !ruby/object:Gem::Version
+        version: '0'
+email: andrew@ankane.org
 executables: []
-
-extensions: []
-
-extra_rdoc_files: 
-- LICENSE.txt
-- README.rdoc
-files: 
-- .document
-- Gemfile
-- Gemfile.lock
+extensions:
+- ext/iceberg/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Cargo.lock
+- Cargo.toml
 - LICENSE.txt
-- README.rdoc
-- Rakefile
-- VERSION
-- iceberg.gemspec
+- NOTICE.txt
+- README.md
+- ext/iceberg/Cargo.toml
+- ext/iceberg/extconf.rb
+- ext/iceberg/src/arrow.rs
+- ext/iceberg/src/catalog.rs
+- ext/iceberg/src/error.rs
+- ext/iceberg/src/lib.rs
+- ext/iceberg/src/runtime.rs
+- ext/iceberg/src/scan.rs
+- ext/iceberg/src/table.rs
+- ext/iceberg/src/utils.rs
 - lib/iceberg.rb
-- test/helper.rb
-- test/test_iceberg.rb
-homepage: http://github.com/stellard/iceberg
-licenses: 
-- MIT
-post_install_message: 
+- lib/iceberg/catalog.rb
+- lib/iceberg/glue_catalog.rb
+- lib/iceberg/memory_catalog.rb
+- lib/iceberg/rest_catalog.rb
+- lib/iceberg/schema.rb
+- lib/iceberg/sql_catalog.rb
+- lib/iceberg/static_table.rb
+- lib/iceberg/table.rb
+- lib/iceberg/table_definition.rb
+- lib/iceberg/version.rb
+homepage: https://github.com/ankane/iceberg-ruby
+licenses:
+- Apache-2.0
+metadata: {}
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: 
-rubygems_version: 1.8.5
-signing_key: 
-specification_version: 3
-summary: just a placeholder for now until public release
-test_files: 
-- test/helper.rb
-- test/test_iceberg.rb
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Apache Iceberg for Ruby
+test_files: []

data/.document

@@ -1,5 +0,0 @@
-lib/**/*.rb
-bin/*
-- 
-features/**/*.feature
-LICENSE.txt

data/Gemfile

@@ -1,13 +0,0 @@
-source "http://rubygems.org"
-# Add dependencies required to use your gem here.
-# Example:
-#   gem "activesupport", ">= 2.3.5"
-
-# Add dependencies to develop your gem here.
-# Include everything needed to run rake, tests, features, etc.
-group :development do
-  gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.0.0"
-  gem "jeweler", "~> 1.5.2"
-  gem "rcov", ">= 0"
-end

data/Gemfile.lock

@@ -1,20 +0,0 @@
-GEM
-  remote: http://rubygems.org/
-  specs:
-    git (1.2.5)
-    jeweler (1.5.2)
-      bundler (~> 1.0.0)
-      git (>= 1.2.5)
-      rake
-    rake (0.9.2)
-    rcov (0.9.9)
-    shoulda (2.11.3)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.0.0)
-  jeweler (~> 1.5.2)
-  rcov
-  shoulda

data/README.rdoc

@@ -1,19 +0,0 @@
-= iceberg
-
-Description goes here.
-
-== Contributing to iceberg
- 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
-* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
-* Fork the project
-* Start a feature/bugfix branch
-* Commit and push until you are happy with your contribution
-* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
-* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
-
-== Copyright
-
-Copyright (c) 2011 stellard. See LICENSE.txt for
-further details.
-