kudu_adapter 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +4 -0
- data/.rubocop.yml +8 -0
- data/Gemfile +9 -0
- data/LICENSE.txt +20 -0
- data/README.md +178 -0
- data/kudu_adapter.gemspec +33 -0
- data/lib/active_record/connection_adapters/kudu/column.rb +17 -0
- data/lib/active_record/connection_adapters/kudu/database_statements.rb +41 -0
- data/lib/active_record/connection_adapters/kudu/quoting.rb +51 -0
- data/lib/active_record/connection_adapters/kudu/schema_creation.rb +89 -0
- data/lib/active_record/connection_adapters/kudu/schema_statements.rb +507 -0
- data/lib/active_record/connection_adapters/kudu/sql_type_metadata.rb +16 -0
- data/lib/active_record/connection_adapters/kudu/table_definition.rb +32 -0
- data/lib/active_record/connection_adapters/kudu/type/big_int.rb +22 -0
- data/lib/active_record/connection_adapters/kudu/type/boolean.rb +23 -0
- data/lib/active_record/connection_adapters/kudu/type/char.rb +17 -0
- data/lib/active_record/connection_adapters/kudu/type/date_time.rb +21 -0
- data/lib/active_record/connection_adapters/kudu/type/double.rb +17 -0
- data/lib/active_record/connection_adapters/kudu/type/float.rb +18 -0
- data/lib/active_record/connection_adapters/kudu/type/integer.rb +22 -0
- data/lib/active_record/connection_adapters/kudu/type/small_int.rb +22 -0
- data/lib/active_record/connection_adapters/kudu/type/string.rb +17 -0
- data/lib/active_record/connection_adapters/kudu/type/time.rb +30 -0
- data/lib/active_record/connection_adapters/kudu/type/tiny_int.rb +22 -0
- data/lib/active_record/connection_adapters/kudu_adapter.rb +173 -0
- data/lib/active_record/tasks/kudu_database_tasks.rb +29 -0
- data/lib/arel/visitors/kudu.rb +7 -0
- data/lib/kudu_adapter/bind_substitution.rb +15 -0
- data/lib/kudu_adapter/table_definition_extensions.rb +28 -0
- data/lib/kudu_adapter/version.rb +5 -0
- data/lib/kudu_adapter.rb +5 -0
- data/spec/spec_config.yaml.template +8 -0
- data/spec/spec_helper.rb +124 -0
- metadata +205 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a16fccf807f95dbd8a16bf90c03314fda9564a6d
|
4
|
+
data.tar.gz: 3f10e1d226527b8550e4950c03632d9d9a2060cb
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 05f9b3a9ac337b3b93bf6ad3f2df444bacf8982ca67fcc61d9c071783732a8a6e2e89d74d3944a7cb1fd18a3768ad830734ece62dcdfbe191deb677450943869
|
7
|
+
data.tar.gz: ce8a364831494992b183ba50af1b2cb922ca154c44b657b688bd69a62e7417c1c57a6e10d5bbb0368c89b47739901069d37d4cced164cfc80281f3caa5bea24c
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2017 OnePageCRM
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,178 @@
|
|
1
|
+
# Info
|
2
|
+
|
3
|
+
Rails db adapter for Impala layer over Kudu database.
|
4
|
+
|
5
|
+
# Usage
|
6
|
+
|
7
|
+
## Requirements
|
8
|
+
|
9
|
+
Active connection to Impala host and port 21000.
|
10
|
+
|
11
|
+
## Config database.yml
|
12
|
+
|
13
|
+
We need to config our yml like:
|
14
|
+
|
15
|
+
```
|
16
|
+
default: &default
|
17
|
+
adapter: kudu
|
18
|
+
host: 'localhost'
|
19
|
+
port: 21000
|
20
|
+
user: 'test'
|
21
|
+
password: 'test'
|
22
|
+
timeout: 5000
|
23
|
+
```
|
24
|
+
|
25
|
+
User and password not required.
|
26
|
+
|
27
|
+
## Supported migration functionality
|
28
|
+
|
29
|
+
With our adapter almost all migration functionality is supported, such as:
|
30
|
+
|
31
|
+
* create_table
|
32
|
+
* remove_table
|
33
|
+
* rename_table
|
34
|
+
* add_column
|
35
|
+
* remove_column
|
36
|
+
* rename_column
|
37
|
+
* and so on...
|
38
|
+
|
39
|
+
## Current limitations
|
40
|
+
|
41
|
+
* Creating any indexes (KUDU supports only primary key fields)
|
42
|
+
|
43
|
+
# Examples
|
44
|
+
|
45
|
+
Here is some examples...
|
46
|
+
|
47
|
+
## Create table
|
48
|
+
|
49
|
+
Because Kudu does not support AUTO INCREMENT INT fields we must ensure any primary key field is created as string field.
|
50
|
+
|
51
|
+
```
|
52
|
+
class CreateUsers < ActiveRecord::Migration[5.1]
|
53
|
+
def change
|
54
|
+
create_table :users, id: false do |t|
|
55
|
+
t.string :id, primary_key: true
|
56
|
+
t.string :account_id, null: false
|
57
|
+
t.string :name, null: false
|
58
|
+
t.string :email, null: false
|
59
|
+
t.string :company_id, null: false
|
60
|
+
t.string :company_name
|
61
|
+
t.timestamps
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
```
|
66
|
+
|
67
|
+
In example above we have only 1 primary key field and our model will work fully functionally when we using update(), delete() methods.
|
68
|
+
|
69
|
+
We can set additional primary key field, like
|
70
|
+
|
71
|
+
```
|
72
|
+
create_table :users, id: false do |t|
|
73
|
+
t.string :id, primary_key: true
|
74
|
+
t.string :account_id, primary_key: true
|
75
|
+
```
|
76
|
+
|
77
|
+
Here, we have two fields in primary key (id, account_id) and with KUDU table will be created with those 2 primary keys, but due to limitation of Rails will not be possible to use model delete(), update() methods.
|
78
|
+
|
79
|
+
## Add new column
|
80
|
+
|
81
|
+
Basic case:
|
82
|
+
|
83
|
+
```
|
84
|
+
class AddZipCodeToUsers < ActiveRecord::Migration[5.1]
|
85
|
+
def change
|
86
|
+
add_column :users, :zip_code, :string
|
87
|
+
end
|
88
|
+
end
|
89
|
+
```
|
90
|
+
|
91
|
+
In case of adding new primary key field, like:
|
92
|
+
|
93
|
+
```
|
94
|
+
class AddCompanyToUsers < ActiveRecord::Migration[5.1]
|
95
|
+
def up
|
96
|
+
add_column :users, :company_id, :string, primary_key: true
|
97
|
+
reload_table_data :users, :company_id, default: '#company-id'
|
98
|
+
end
|
99
|
+
def down
|
100
|
+
remove_column :users, :company_id
|
101
|
+
end
|
102
|
+
end
|
103
|
+
```
|
104
|
+
|
105
|
+
we will initialize specialized method at migration side, and basically this will happen:
|
106
|
+
|
107
|
+
* New table "table_name_redefined" will be created based on original table name (ex. users -> users_redefined) with included new field, new primary key field.
|
108
|
+
* Old table "users" will be renamed to "users_temp"
|
109
|
+
* Data will be copied from users_temp => users with additional new field and default value
|
110
|
+
* Old temporary table "users_temp" will be deleted
|
111
|
+
|
112
|
+
## Delete column
|
113
|
+
|
114
|
+
Basic case:
|
115
|
+
|
116
|
+
```
|
117
|
+
class RemoveZipCodeFromUsers < ActiveRecord::Migration[5.1]
|
118
|
+
def change
|
119
|
+
remove_column :users, :zip_code
|
120
|
+
end
|
121
|
+
end
|
122
|
+
```
|
123
|
+
|
124
|
+
In case of deleting primary key field (existing) procedure is same like for adding new column with primary key.
|
125
|
+
|
126
|
+
## Model associations
|
127
|
+
|
128
|
+
Model associations will work without foreign keys, like:
|
129
|
+
|
130
|
+
```
|
131
|
+
class CreateAccounts < ActiveRecord::Migration[5.1]
|
132
|
+
def change
|
133
|
+
create_table :accounts, id: false do |t|
|
134
|
+
t.string :id, primary_key: true
|
135
|
+
t.boolean :is_active, default: true
|
136
|
+
t.timestamps
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
class Account < ApplicationRecord
|
142
|
+
has_many :users, foreign_key: 'account_id'
|
143
|
+
end
|
144
|
+
```
|
145
|
+
|
146
|
+
and table Users with following construct...
|
147
|
+
|
148
|
+
```
|
149
|
+
class CreateUsers < ActiveRecord::Migration[5.1]
|
150
|
+
def change
|
151
|
+
create_table :users, id: false do |t|
|
152
|
+
t.string :id, primary_key: true
|
153
|
+
t.string :account_id, null: false
|
154
|
+
t.string :name, null: false
|
155
|
+
t.string :email, null: false
|
156
|
+
t.string :company_id, null: false
|
157
|
+
t.string :company_name
|
158
|
+
t.timestamps
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
class User < ApplicationRecord
|
164
|
+
belongs_to :account, primary_key: 'id'
|
165
|
+
end
|
166
|
+
```
|
167
|
+
|
168
|
+
This way we're able to do
|
169
|
+
|
170
|
+
```
|
171
|
+
Account.first.users => [#User, #User, ...]
|
172
|
+
```
|
173
|
+
|
174
|
+
or
|
175
|
+
|
176
|
+
```
|
177
|
+
User.first.account => #Account
|
178
|
+
```
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$LOAD_PATH.push File.expand_path('../lib', __FILE__)
|
4
|
+
require 'kudu_adapter/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = 'kudu_adapter'
|
8
|
+
s.version = KuduAdapter::VERSION
|
9
|
+
s.platform = Gem::Platform::RUBY
|
10
|
+
s.required_ruby_version = '>= 2.4.0'
|
11
|
+
s.authors = ['Paweł Smoliński', 'OnePageCRM']
|
12
|
+
s.licenses = ['MIT']
|
13
|
+
s.email = 'devteam@onepagecrm.com'
|
14
|
+
s.homepage = 'https://github.com/OnePageCRM/kudu_adapter'
|
15
|
+
s.summary = "ActiveRecord adapter for Cloudera's Kudu over Impala database"
|
16
|
+
s.description = "ActiveRecord adapter for Cloudera's Kudu over Impala database"
|
17
|
+
s.email = 'devteam@onepagecrm.com'
|
18
|
+
|
19
|
+
s.add_runtime_dependency('arel', ['~> 8.0'])
|
20
|
+
s.add_runtime_dependency('activemodel', ['~> 5.1.0'])
|
21
|
+
s.add_runtime_dependency('activerecord', ['~> 5.1.0'])
|
22
|
+
s.add_runtime_dependency('impala', ['~> 0.5.1'])
|
23
|
+
|
24
|
+
s.add_development_dependency('bundler')
|
25
|
+
s.add_development_dependency('rake')
|
26
|
+
s.add_development_dependency('rspec')
|
27
|
+
s.add_development_dependency('pry')
|
28
|
+
s.add_development_dependency('rubocop')
|
29
|
+
|
30
|
+
s.files = `git ls-files`.split("\n")
|
31
|
+
s.test_files = `git ls-files -- spec/*`.split("\n")
|
32
|
+
s.require_paths = ['lib']
|
33
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_record/connection_adapters/column'
|
4
|
+
|
5
|
+
# :nodoc:
|
6
|
+
module ActiveRecord
|
7
|
+
module ConnectionAdapters
|
8
|
+
module Kudu
|
9
|
+
# :nodoc:
|
10
|
+
class Column < ::ActiveRecord::ConnectionAdapters::Column
|
11
|
+
def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, comment = nil)
|
12
|
+
super(name, default, sql_type_metadata, null, table_name, nil, nil, comment: comment)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_record/result'
|
4
|
+
|
5
|
+
module ActiveRecord
|
6
|
+
module ConnectionAdapters
|
7
|
+
module Kudu
|
8
|
+
# :nodoc:
|
9
|
+
module DatabaseStatements
|
10
|
+
# :nodoc:
|
11
|
+
def exec_query(sql, _ = 'SQL', binds = [], prepare: false)
|
12
|
+
::Rails.logger.warn 'Prepared statements are not supported' if prepare
|
13
|
+
|
14
|
+
unless without_prepared_statement? binds
|
15
|
+
type_casted_binds(binds).each do |bind|
|
16
|
+
bind = quote(bind)
|
17
|
+
sql = sql.sub('?', bind.to_s)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
#::Rails.logger.info 'QUERY : ' + sql.to_s
|
21
|
+
result = connection.query sql
|
22
|
+
columns = result.first&.keys.to_a
|
23
|
+
rows = result.map { |row| row.fetch_values(*columns) }
|
24
|
+
::ActiveRecord::Result.new(columns.map(&:to_s), rows)
|
25
|
+
end
|
26
|
+
|
27
|
+
def exec_delete(sql, name, binds)
|
28
|
+
# We are not able to return number of affected rows so we will just say that there was some update
|
29
|
+
super
|
30
|
+
1
|
31
|
+
end
|
32
|
+
|
33
|
+
def exec_update(sql, name, binds)
|
34
|
+
# We are not able to return number of affected rows so we will just say that there was some update
|
35
|
+
super
|
36
|
+
1
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/big_decimal/conversions'
|
4
|
+
require 'active_support/multibyte/chars'
|
5
|
+
|
6
|
+
module ActiveRecord
|
7
|
+
module ConnectionAdapters # :nodoc:
|
8
|
+
module Kudu
|
9
|
+
module Quoting
|
10
|
+
|
11
|
+
QUOTED_TRUE, QUOTED_FALSE = true.to_s, false.to_s
|
12
|
+
|
13
|
+
def quote_column_name(column_name)
|
14
|
+
column_name.to_s
|
15
|
+
end
|
16
|
+
|
17
|
+
def quote_table_name(table_name)
|
18
|
+
quote_column_name table_name
|
19
|
+
end
|
20
|
+
|
21
|
+
def quote_default_expression(value, column) # :nodoc:
|
22
|
+
if value.is_a?(Proc)
|
23
|
+
value.call
|
24
|
+
else
|
25
|
+
value = lookup_cast_type(column.sql_type).serialize(value)
|
26
|
+
# DOUBLE, FLOAT represented as 0.0 but KUDU supports only DEFAULT statement as 0
|
27
|
+
value = value.to_i if %w(DOUBLE FLOAT).include? column.sql_type
|
28
|
+
quote(value)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def quoted_true
|
33
|
+
QUOTED_TRUE
|
34
|
+
end
|
35
|
+
|
36
|
+
def unquoted_true
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
def quoted_false
|
41
|
+
QUOTED_FALSE
|
42
|
+
end
|
43
|
+
|
44
|
+
def unquoted_false
|
45
|
+
false
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_record/connection_adapters/abstract/schema_creation'
|
4
|
+
|
5
|
+
module ActiveRecord
|
6
|
+
module ConnectionAdapters
|
7
|
+
module Kudu
|
8
|
+
# :nodoc:
|
9
|
+
class SchemaCreation < ::ActiveRecord::ConnectionAdapters::AbstractAdapter::SchemaCreation
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def visit_AddColumnDefinition(obj)
|
14
|
+
"ADD COLUMNS (#{accept(obj.column)})"
|
15
|
+
end
|
16
|
+
|
17
|
+
def visit_ColumnDefinition(obj)
|
18
|
+
obj.sql_type = type_to_sql(obj.type, obj.options)
|
19
|
+
column_sql = "#{quote_column_name(obj.name)} #{obj.sql_type}".dup
|
20
|
+
add_column_options!(column_sql, column_options(obj))
|
21
|
+
end
|
22
|
+
|
23
|
+
# @param table_def [::ActiveRecord::ConnectionAdapters::Kudu::TableDefinition]
|
24
|
+
def visit_TableDefinition(table_def)
|
25
|
+
create_sql = "CREATE#{' EXTERNAL' if table_def.external} TABLE #{quote_table_name(table_def.name)} "
|
26
|
+
|
27
|
+
statements = table_def.columns.map { |col| accept col }
|
28
|
+
|
29
|
+
primary_keys = if table_def.primary_keys&.any?
|
30
|
+
table_def.primary_keys
|
31
|
+
else
|
32
|
+
table_def.columns.select { |col| col.options[:primary_key] }.map(&:name)
|
33
|
+
end
|
34
|
+
|
35
|
+
raise "Table #{table_def.name} does not have primary key(s) defined" if primary_keys.empty?
|
36
|
+
quoted_names = primary_keys.map { |pk| quote_column_name(pk) }
|
37
|
+
|
38
|
+
statements << "PRIMARY KEY (#{quoted_names.join(', ')})"
|
39
|
+
|
40
|
+
create_sql += "(#{statements.join(', ')})" if statements.present?
|
41
|
+
add_table_options!(create_sql, table_options(table_def))
|
42
|
+
|
43
|
+
# For managed Kudu tables partitioning must be defined
|
44
|
+
unless table_def.external
|
45
|
+
# If no partition columns will be provided, we will use all primary keys defined
|
46
|
+
partition_columns = table_def.partition_columns || primary_keys
|
47
|
+
if (partition_columns - table_def.columns.map(&:name)).any?
|
48
|
+
raise 'Non-existing columns have been selected as partition indicators'
|
49
|
+
end
|
50
|
+
|
51
|
+
partitions_count = table_def.partitions_count || 2
|
52
|
+
quoted_names = partition_columns.map { |pc| quote_column_name(pc) }
|
53
|
+
create_sql += " PARTITION BY HASH(#{quoted_names.join(', ')}) PARTITIONS #{partitions_count.to_i}"
|
54
|
+
# TODO: partitions range
|
55
|
+
end
|
56
|
+
|
57
|
+
create_sql + ' STORED AS KUDU'
|
58
|
+
end
|
59
|
+
|
60
|
+
def add_column_options!(sql, options)
|
61
|
+
# [NOT] NULL
|
62
|
+
if options[:primary_key]
|
63
|
+
sql += ' NOT NULL'
|
64
|
+
else
|
65
|
+
options[:null] = true if options[:null].nil?
|
66
|
+
sql += options[:null] ? ' NULL' : ' NOT NULL'
|
67
|
+
end
|
68
|
+
|
69
|
+
# Encodings:
|
70
|
+
# AUTO_ENCODING, PLAIN_ENCODING, RLE, DICT_ENCODING, BIT_SHUFFLE, PREFIX_ENCODING
|
71
|
+
sql += " ENCODING #{options[:encoding].to_s}" if options[:encoding]
|
72
|
+
|
73
|
+
# Compressions:
|
74
|
+
# LZ4, SNAPPY, and ZLIB
|
75
|
+
sql += " COMPRESSION #{options[:compression].to_s}" if options[:compression]
|
76
|
+
|
77
|
+
# Default values
|
78
|
+
sql += " DEFAULT #{quote_default_expression(options[:default], options[:column])}" unless options[:default].nil?
|
79
|
+
|
80
|
+
# Block size
|
81
|
+
sql += " BLOCK SIZE #{options[:block_size].to_i}" if options[:block_size]
|
82
|
+
|
83
|
+
sql
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|