dwh 0.1.0

data/lib/dwh/table.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative 'column'
+
+module DWH
+  # Container to map to a data warehouse table.
+  # If you initialize with a fuly qualified table name
+  # , it will automatically create catalog and schema components.
+  #
+  # This is the object returned from +metadata+ method call of an adapter
+  #
+  # ==== Examples
+  #   Table.new("dwh.public.hello_world_table")
+  #
+  #   table_stats_instance = adapter.stats("my_table", schema: "dwh")
+  #   Table.new("my_table", schema: "dwh", stats: table_stats_instance)
+  class Table
+    attr_reader :physical_name, :schema, :catalog, :columns, :table_stats
+
+    def initialize(physical_name, schema: nil, catalog: nil, table_stats: nil)
+      parts = physical_name.split('.')
+
+      @physical_name = parts.last
+      @table_stats = table_stats
+      @catalog = catalog
+      @schema = schema
+
+      @catalog = parts.first if @catalog.nil? && parts.length > 2
+
+      if @schema.nil?
+        if parts.length == 2
+          @schema = parts.first
+        elsif parts.length > 2
+          @schema = parts[1]
+        end
+      end
+
+      @columns = []
+    end
+
+    def <<(column)
+      @columns << column
+    end
+
+    def fully_qualified_table_name
+      [catalog, schema, physical_name].compact.join('.')
+    end
+
+    def fully_qualified_schema_name
+      [catalog, schema].compact.join('.')
+    end
+
+    def catalog_and_schema?
+      catalog && schema
+    end
+
+    def catalog_or_schema?
+      catalog || schema
+    end
+
+    def stats
+      @table_stats
+    end
+
+    def to_h
+      {
+        physical_name: physical_name,
+        schema: schema,
+        catalog: catalog,
+        columns: columns.map(&:to_h),
+        stats: table_stats&.to_h
+      }
+    end
+
+    def size
+      @table_stats&.row_count || 0
+    end
+
+    def find_column(name)
+      columns.find { |c| c.name.downcase == name.downcase }
+    end
+
+    def self.from_hash_or_json(physical_name, metadata)
+      metadata = JSON.parse(metadata) if metadata.is_a?(String)
+      metadata.symbolize_keys!
+
+      stats = TableStats.new(**metadata[:stats].symbolize_keys) if metadata.key?(:stats)
+      table = new(physical_name, table_stats: stats)
+
+      metadata[:columns]&.each do |col|
+        col.symbolize_keys!
+        table << Column.new(
+          name: col[:name],
+          data_type: col[:data_type],
+          precision: col[:precision],
+          scale: col[:scale],
+          max_char_length: col[:max_char_length],
+          schema_type: col[:schema_type]
+        )
+      end
+
+      table
+    end
+  end
+end

data/lib/dwh/table_stats.rb

@@ -0,0 +1,51 @@
+module DWH
+  # TableStats is returned when calling Adapter#stat.
+  # This currently ust provide basic stats,
+  # but could be enhanced in the future to provide more
+  # introspection on the table. For examples, indexes and
+  # partition settings. Cardinality of various columns
+  # etc.
+  class TableStats
+    # @return [Integer] total rows in table
+    attr_accessor :row_count
+
+    # @return [DateTime] when a date column is passed to {Adapters::Adapter#stats} it
+    #   will return date of first record in the table
+    attr_accessor :date_start
+
+    # @return [DateTime] when a date column is passed to {Adapters::Adapter#stats} it
+    #   returns the date of the last record in the table
+    attr_accessor :date_end
+
+    def initialize(row_count: nil, date_start: nil, date_end: nil)
+      @row_count = row_count.nil? ? 0 : row_count.to_i
+      @date_start = parse_date(date_start)
+      @date_end = parse_date(date_end)
+    end
+
+    # Hash of the stats attributes
+    # @return [Hash] of the attributes
+    def to_h
+      {
+        row_count: row_count,
+        date_start: date_start,
+        date_end: date_end
+      }
+    end
+
+    private
+
+    def parse_date(date)
+      case date
+      when nil
+        date
+      when String
+        DateTime.parse(date)
+      when Time, Date
+        DateTime.parse(date.to_s)
+      else
+        raise ConfigError, "Unexpected date class: #{date.class.name}"
+      end
+    end
+  end
+end

data/lib/dwh/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DWH
+  VERSION = '0.1.0'
+end

data/lib/dwh.rb

@@ -0,0 +1,54 @@
+require 'faraday'
+require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/object/blank'
+require 'active_support/duration'
+
+require_relative 'dwh/version'
+require_relative 'dwh/errors'
+require_relative 'dwh/logger'
+require_relative 'dwh/streaming_stats'
+require_relative 'dwh/factory'
+require_relative 'dwh/adapters'
+require_relative 'dwh/table'
+require_relative 'dwh/table_stats'
+require_relative 'dwh/adapters/druid'
+require_relative 'dwh/adapters/trino'
+require_relative 'dwh/adapters/postgres'
+require_relative 'dwh/adapters/snowflake'
+require_relative 'dwh/adapters/my_sql'
+require_relative 'dwh/adapters/sql_server'
+require_relative 'dwh/adapters/duck_db'
+require_relative 'dwh/adapters/athena'
+
+# DWH encapsulates the full functionality of this gem.
+#
+# @example Create an instance of an existing registered adapter:
+#   DWH.create("snowflake", {warehouse: "wh", account_id: "myid"})
+#
+# @example Check if an adapter exists:
+#   DWH.adapter?(:redshift)
+#
+# @example Register your own adatper:
+#   DWH.register(:my_adapter, MyLib::MyAdapter)
+module DWH
+  INT_TYPES = %w[int integer bigint tinyint smallint].freeze
+  DEC_TYPES = %w[real float double decimal].freeze
+  STRING_TYPES = %w[string char varchar varbinary json].freeze
+  TIMESTAMP_TYPES = ['timestamp with time zone', 'timestamp(p)', 'timestamp'].freeze
+  DATE_TYPES = %w[date].freeze
+
+  extend Factory
+
+  # Register default adapters
+  register(:druid, Adapters::Druid)
+  register(:postgres, Adapters::Postgres)
+  register(:trino, Adapters::Trino)
+  register(:snowflake, Adapters::Snowflake)
+  register(:mysql, Adapters::MySql)
+  register(:sqlserver, Adapters::SqlServer)
+  register(:duckdb, Adapters::DuckDb)
+  register(:athena, Adapters::Athena)
+
+  # start_reaper
+end

data/sig/dwh.rbs

@@ -0,0 +1,4 @@
+module Dwh
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,231 @@
+--- !ruby/object:Gem::Specification
+name: dwh
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ajo Abraham
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 8.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 8.0.2
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.5
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jwt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.1
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  Provides a unified interface across data warehouses to connect, execute, and introspect. This is not an ORM but a fast
+  integrationg solution. It is quite easy to add new database adapters. Supports popular cloud warehouses too.
+email:
+- ajo@strata.site
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE
+- README.md
+- Rakefile
+- docs/DWH.html
+- docs/DWH/Adapters.html
+- docs/DWH/Adapters/Adapter.html
+- docs/DWH/Adapters/Athena.html
+- docs/DWH/Adapters/Boolean.html
+- docs/DWH/Adapters/Druid.html
+- docs/DWH/Adapters/DuckDb.html
+- docs/DWH/Adapters/MySql.html
+- docs/DWH/Adapters/OpenAuthorizable.html
+- docs/DWH/Adapters/OpenAuthorizable/ClassMethods.html
+- docs/DWH/Adapters/Postgres.html
+- docs/DWH/Adapters/Snowflake.html
+- docs/DWH/Adapters/SqlServer.html
+- docs/DWH/Adapters/Trino.html
+- docs/DWH/AuthenticationError.html
+- docs/DWH/Behaviors.html
+- docs/DWH/Capabilities.html
+- docs/DWH/Column.html
+- docs/DWH/ConfigError.html
+- docs/DWH/ConnectionError.html
+- docs/DWH/DWHError.html
+- docs/DWH/ExecutionError.html
+- docs/DWH/Factory.html
+- docs/DWH/Functions.html
+- docs/DWH/Functions/Arrays.html
+- docs/DWH/Functions/Dates.html
+- docs/DWH/Functions/ExtractDatePart.html
+- docs/DWH/Functions/Nulls.html
+- docs/DWH/Logger.html
+- docs/DWH/OAuthError.html
+- docs/DWH/Settings.html
+- docs/DWH/StreamingStats.html
+- docs/DWH/Table.html
+- docs/DWH/TableStats.html
+- docs/DWH/TokenExpiredError.html
+- docs/DWH/UnsupportedCapability.html
+- docs/_index.html
+- docs/class_list.html
+- docs/css/common.css
+- docs/css/full_list.css
+- docs/css/style.css
+- docs/file.README.html
+- docs/file.adapters.html
+- docs/file.creating-adapters.html
+- docs/file.getting-started.html
+- docs/file.usage.html
+- docs/file_list.html
+- docs/frames.html
+- docs/guides/adapters.md
+- docs/guides/creating-adapters.md
+- docs/guides/getting-started.md
+- docs/guides/usage.md
+- docs/index.html
+- docs/js/app.js
+- docs/js/full_list.js
+- docs/js/jquery.js
+- docs/method_list.html
+- docs/top-level-namespace.html
+- lib/dwh.rb
+- lib/dwh/adapters.rb
+- lib/dwh/adapters/athena.rb
+- lib/dwh/adapters/druid.rb
+- lib/dwh/adapters/duck_db.rb
+- lib/dwh/adapters/my_sql.rb
+- lib/dwh/adapters/open_authorizable.rb
+- lib/dwh/adapters/postgres.rb
+- lib/dwh/adapters/snowflake.rb
+- lib/dwh/adapters/sql_server.rb
+- lib/dwh/adapters/trino.rb
+- lib/dwh/behaviors.rb
+- lib/dwh/capabilities.rb
+- lib/dwh/column.rb
+- lib/dwh/errors.rb
+- lib/dwh/factory.rb
+- lib/dwh/functions.rb
+- lib/dwh/functions/arrays.rb
+- lib/dwh/functions/dates.rb
+- lib/dwh/functions/extract_date_part.rb
+- lib/dwh/functions/nulls.rb
+- lib/dwh/logger.rb
+- lib/dwh/settings.rb
+- lib/dwh/settings/athena.yml
+- lib/dwh/settings/base.yml
+- lib/dwh/settings/databricks.yml
+- lib/dwh/settings/druid.yml
+- lib/dwh/settings/duckdb.yml
+- lib/dwh/settings/mysql.yml
+- lib/dwh/settings/postgres.yml
+- lib/dwh/settings/redshift.yml
+- lib/dwh/settings/snowflake.yml
+- lib/dwh/settings/sqlserver.yml
+- lib/dwh/settings/trino.yml
+- lib/dwh/streaming_stats.rb
+- lib/dwh/table.rb
+- lib/dwh/table_stats.rb
+- lib/dwh/version.rb
+- sig/dwh.rbs
+homepage: https://www.strata.site
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://www.strata.site
+  source_code_uri: https://github.com/stratasite/dwh.git
+  changelog_uri: https://github.com/stratasite/dwh/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.4.4
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Data warehouse adapters for interacting with popular data warehouses.
+test_files: []