kudu_adapter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a16fccf807f95dbd8a16bf90c03314fda9564a6d
+  data.tar.gz: 3f10e1d226527b8550e4950c03632d9d9a2060cb
+SHA512:
+  metadata.gz: 05f9b3a9ac337b3b93bf6ad3f2df444bacf8982ca67fcc61d9c071783732a8a6e2e89d74d3944a7cb1fd18a3768ad830734ece62dcdfbe191deb677450943869
+  data.tar.gz: ce8a364831494992b183ba50af1b2cb922ca154c44b657b688bd69a62e7417c1c57a6e10d5bbb0368c89b47739901069d37d4cced164cfc80281f3caa5bea24c

data/.gitignore

@@ -0,0 +1,4 @@
+syntax: glob
+.idea/
+Gemfile.lock
+spec_config.yaml

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  TargetRubyVersion: 2.4
+  Exclude:
+    - '**/schema_creation.rb'
+
+Metrics/LineLength:
+  Enabled: true
+  Max: 120

data/Gemfile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'simplecov', require: false
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2017 OnePageCRM
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,178 @@
+# Info
+
+Rails db adapter for Impala layer over Kudu database.
+
+# Usage
+
+## Requirements
+
+Active connection to Impala host and port 21000.
+
+## Config database.yml
+
+We need to config our yml like:
+
+```
+default: &default
+  adapter: kudu
+  host: 'localhost'
+  port: 21000
+  user: 'test'
+  password: 'test'
+  timeout: 5000
+```
+
+User and password not required.
+
+## Supported migration functionality
+
+With our adapter almost all migration functionality is supported, such as:
+
+* create_table
+* remove_table
+* rename_table
+* add_column
+* remove_column
+* rename_column
+* and so on...
+
+## Current limitations
+
+* Creating any indexes (KUDU supports only primary key fields)
+
+# Examples
+
+Here is some examples...
+
+## Create table
+
+Because Kudu does not support AUTO INCREMENT INT fields we must ensure any primary key field is created as string field.
+
+```
+class CreateUsers < ActiveRecord::Migration[5.1]
+  def change
+    create_table :users, id: false do |t|
+      t.string :id, primary_key: true
+      t.string :account_id, null: false
+      t.string :name, null: false
+      t.string :email, null: false
+      t.string :company_id, null: false
+      t.string :company_name
+      t.timestamps
+    end
+  end
+end
+```
+
+In example above we have only 1 primary key field and our model will work fully functionally when we using update(), delete() methods.
+
+We can set additional primary key field, like
+
+```
+create_table :users, id: false do |t|
+      t.string :id, primary_key: true
+      t.string :account_id, primary_key: true
+```
+
+Here, we have two fields in primary key (id, account_id) and with KUDU table will be created with those 2 primary keys, but due to limitation of Rails will not be possible to use model delete(), update() methods.
+
+## Add new column
+
+Basic case:
+
+```
+class AddZipCodeToUsers < ActiveRecord::Migration[5.1]
+  def change
+    add_column :users, :zip_code, :string
+  end
+end
+```
+
+In case of adding new primary key field, like:
+
+```
+class AddCompanyToUsers < ActiveRecord::Migration[5.1]
+  def up
+    add_column :users, :company_id, :string, primary_key: true
+    reload_table_data :users, :company_id, default: '#company-id'
+  end
+  def down
+    remove_column :users, :company_id
+  end
+end
+```
+
+we will initialize specialized method at migration side, and basically this will happen:
+
+* New table "table_name_redefined" will be created based on original table name (ex. users -> users_redefined) with included new field, new primary key field.
+* Old table "users" will be renamed to "users_temp"
+* Data will be copied from users_temp => users with additional new field and default value
+* Old temporary table "users_temp" will be deleted
+
+## Delete column
+
+Basic case:
+
+```
+class RemoveZipCodeFromUsers < ActiveRecord::Migration[5.1]
+  def change
+    remove_column :users, :zip_code
+  end
+end
+```
+
+In case of deleting primary key field (existing) procedure is same like for adding new column with primary key.
+
+## Model associations
+
+Model associations will work without foreign keys, like:
+
+```
+class CreateAccounts < ActiveRecord::Migration[5.1]
+  def change
+    create_table :accounts, id: false do |t|
+      t.string :id, primary_key: true
+      t.boolean :is_active, default: true
+      t.timestamps
+    end
+  end
+end
+
+class Account < ApplicationRecord
+  has_many :users, foreign_key: 'account_id'
+end
+```
+
+and table Users with following construct...
+
+```
+class CreateUsers < ActiveRecord::Migration[5.1]
+  def change
+    create_table :users, id: false do |t|
+      t.string :id, primary_key: true
+      t.string :account_id, null: false
+      t.string :name, null: false
+      t.string :email, null: false
+      t.string :company_id, null: false
+      t.string :company_name
+      t.timestamps
+    end
+  end
+end
+
+class User < ApplicationRecord
+  belongs_to :account, primary_key: 'id'
+end
+```
+
+This way we're able to do
+
+```
+Account.first.users => [#User, #User, ...]
+```
+
+or
+
+```
+User.first.account => #Account
+```

data/kudu_adapter.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'kudu_adapter/version'
+
+Gem::Specification.new do |s|
+  s.name = 'kudu_adapter'
+  s.version = KuduAdapter::VERSION
+  s.platform = Gem::Platform::RUBY
+  s.required_ruby_version = '>= 2.4.0'
+  s.authors = ['Paweł Smoliński', 'OnePageCRM']
+  s.licenses = ['MIT']
+  s.email = 'devteam@onepagecrm.com'
+  s.homepage = 'https://github.com/OnePageCRM/kudu_adapter'
+  s.summary = "ActiveRecord adapter for Cloudera's Kudu over Impala database"
+  s.description = "ActiveRecord adapter for Cloudera's Kudu over Impala database"
+  s.email = 'devteam@onepagecrm.com'
+
+  s.add_runtime_dependency('arel', ['~> 8.0'])
+  s.add_runtime_dependency('activemodel', ['~> 5.1.0'])
+  s.add_runtime_dependency('activerecord', ['~> 5.1.0'])
+  s.add_runtime_dependency('impala', ['~> 0.5.1'])
+
+  s.add_development_dependency('bundler')
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rspec')
+  s.add_development_dependency('pry')
+  s.add_development_dependency('rubocop')
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.require_paths = ['lib']
+end

data/lib/active_record/connection_adapters/kudu/column.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'active_record/connection_adapters/column'
+
+# :nodoc:
+module ActiveRecord
+  module ConnectionAdapters
+    module Kudu
+      # :nodoc:
+      class Column < ::ActiveRecord::ConnectionAdapters::Column
+        def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, comment = nil)
+          super(name, default, sql_type_metadata, null, table_name, nil, nil, comment: comment)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/kudu/database_statements.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'active_record/result'
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Kudu
+      # :nodoc:
+      module DatabaseStatements
+        # :nodoc:
+        def exec_query(sql, _ = 'SQL', binds = [], prepare: false)
+          ::Rails.logger.warn 'Prepared statements are not supported' if prepare
+
+          unless without_prepared_statement? binds
+            type_casted_binds(binds).each do |bind|
+              bind = quote(bind)
+              sql = sql.sub('?', bind.to_s)
+            end
+          end
+          #::Rails.logger.info 'QUERY : ' + sql.to_s
+          result = connection.query sql
+          columns = result.first&.keys.to_a
+          rows = result.map { |row| row.fetch_values(*columns) }
+          ::ActiveRecord::Result.new(columns.map(&:to_s), rows)
+        end
+
+        def exec_delete(sql, name, binds)
+          # We are not able to return number of affected rows so we will just say that there was some update
+          super
+          1
+        end
+
+        def exec_update(sql, name, binds)
+          # We are not able to return number of affected rows so we will just say that there was some update
+          super
+          1
+        end
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/kudu/quoting.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/big_decimal/conversions'
+require 'active_support/multibyte/chars'
+
+module ActiveRecord
+  module ConnectionAdapters # :nodoc:
+    module Kudu
+      module Quoting
+
+        QUOTED_TRUE, QUOTED_FALSE = true.to_s, false.to_s
+
+        def quote_column_name(column_name)
+          column_name.to_s
+        end
+
+        def quote_table_name(table_name)
+          quote_column_name table_name
+        end
+
+        def quote_default_expression(value, column) # :nodoc:
+          if value.is_a?(Proc)
+            value.call
+          else
+            value = lookup_cast_type(column.sql_type).serialize(value)
+            # DOUBLE, FLOAT represented as 0.0 but KUDU supports only DEFAULT statement as 0
+            value = value.to_i if %w(DOUBLE FLOAT).include? column.sql_type
+            quote(value)
+          end
+        end
+
+        def quoted_true
+          QUOTED_TRUE
+        end
+
+        def unquoted_true
+          true
+        end
+
+        def quoted_false
+          QUOTED_FALSE
+        end
+
+        def unquoted_false
+          false
+        end
+
+      end
+    end
+  end
+end

data/lib/active_record/connection_adapters/kudu/schema_creation.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'active_record/connection_adapters/abstract/schema_creation'
+
+module ActiveRecord
+  module ConnectionAdapters
+    module Kudu
+      # :nodoc:
+      class SchemaCreation < ::ActiveRecord::ConnectionAdapters::AbstractAdapter::SchemaCreation
+
+        private
+
+        def visit_AddColumnDefinition(obj)
+          "ADD COLUMNS (#{accept(obj.column)})"
+        end
+
+        def visit_ColumnDefinition(obj)
+          obj.sql_type = type_to_sql(obj.type, obj.options)
+          column_sql = "#{quote_column_name(obj.name)} #{obj.sql_type}".dup
+          add_column_options!(column_sql, column_options(obj))
+        end
+
+        # @param table_def [::ActiveRecord::ConnectionAdapters::Kudu::TableDefinition]
+        def visit_TableDefinition(table_def)
+          create_sql = "CREATE#{' EXTERNAL' if table_def.external} TABLE #{quote_table_name(table_def.name)} "
+
+          statements = table_def.columns.map { |col| accept col }
+
+          primary_keys = if table_def.primary_keys&.any?
+                           table_def.primary_keys
+                         else
+                           table_def.columns.select { |col| col.options[:primary_key] }.map(&:name)
+                         end
+
+          raise "Table #{table_def.name} does not have primary key(s) defined" if primary_keys.empty?
+          quoted_names = primary_keys.map { |pk| quote_column_name(pk) }
+
+          statements << "PRIMARY KEY (#{quoted_names.join(', ')})"
+
+          create_sql += "(#{statements.join(', ')})" if statements.present?
+          add_table_options!(create_sql, table_options(table_def))
+
+          # For managed Kudu tables partitioning must be defined
+          unless table_def.external
+            # If no partition columns will be provided, we will use all primary keys defined
+            partition_columns = table_def.partition_columns || primary_keys
+            if (partition_columns - table_def.columns.map(&:name)).any?
+              raise 'Non-existing columns have been selected as partition indicators'
+            end
+
+            partitions_count = table_def.partitions_count || 2
+            quoted_names = partition_columns.map { |pc| quote_column_name(pc) }
+            create_sql += " PARTITION BY HASH(#{quoted_names.join(', ')}) PARTITIONS #{partitions_count.to_i}"
+            # TODO: partitions range
+          end
+
+          create_sql + ' STORED AS KUDU'
+        end
+
+        def add_column_options!(sql, options)
+          # [NOT] NULL
+          if options[:primary_key]
+            sql += ' NOT NULL'
+          else
+            options[:null] = true if options[:null].nil?
+            sql += options[:null] ? ' NULL' : ' NOT NULL'
+          end
+
+          # Encodings:
+          #  AUTO_ENCODING, PLAIN_ENCODING, RLE, DICT_ENCODING, BIT_SHUFFLE, PREFIX_ENCODING
+          sql += " ENCODING #{options[:encoding].to_s}" if options[:encoding]
+
+          # Compressions:
+          #  LZ4, SNAPPY, and ZLIB
+          sql += " COMPRESSION #{options[:compression].to_s}" if options[:compression]
+
+          # Default values
+          sql += " DEFAULT #{quote_default_expression(options[:default], options[:column])}" unless options[:default].nil?
+
+          # Block size
+          sql += " BLOCK SIZE #{options[:block_size].to_i}" if options[:block_size]
+
+          sql
+        end
+
+      end
+    end
+  end
+end