kudu_adapter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +4 -0
- data/.rubocop.yml +8 -0
- data/Gemfile +9 -0
- data/LICENSE.txt +20 -0
- data/README.md +178 -0
- data/kudu_adapter.gemspec +33 -0
- data/lib/active_record/connection_adapters/kudu/column.rb +17 -0
- data/lib/active_record/connection_adapters/kudu/database_statements.rb +41 -0
- data/lib/active_record/connection_adapters/kudu/quoting.rb +51 -0
- data/lib/active_record/connection_adapters/kudu/schema_creation.rb +89 -0
- data/lib/active_record/connection_adapters/kudu/schema_statements.rb +507 -0
- data/lib/active_record/connection_adapters/kudu/sql_type_metadata.rb +16 -0
- data/lib/active_record/connection_adapters/kudu/table_definition.rb +32 -0
- data/lib/active_record/connection_adapters/kudu/type/big_int.rb +22 -0
- data/lib/active_record/connection_adapters/kudu/type/boolean.rb +23 -0
- data/lib/active_record/connection_adapters/kudu/type/char.rb +17 -0
- data/lib/active_record/connection_adapters/kudu/type/date_time.rb +21 -0
- data/lib/active_record/connection_adapters/kudu/type/double.rb +17 -0
- data/lib/active_record/connection_adapters/kudu/type/float.rb +18 -0
- data/lib/active_record/connection_adapters/kudu/type/integer.rb +22 -0
- data/lib/active_record/connection_adapters/kudu/type/small_int.rb +22 -0
- data/lib/active_record/connection_adapters/kudu/type/string.rb +17 -0
- data/lib/active_record/connection_adapters/kudu/type/time.rb +30 -0
- data/lib/active_record/connection_adapters/kudu/type/tiny_int.rb +22 -0
- data/lib/active_record/connection_adapters/kudu_adapter.rb +173 -0
- data/lib/active_record/tasks/kudu_database_tasks.rb +29 -0
- data/lib/arel/visitors/kudu.rb +7 -0
- data/lib/kudu_adapter/bind_substitution.rb +15 -0
- data/lib/kudu_adapter/table_definition_extensions.rb +28 -0
- data/lib/kudu_adapter/version.rb +5 -0
- data/lib/kudu_adapter.rb +5 -0
- data/spec/spec_config.yaml.template +8 -0
- data/spec/spec_helper.rb +124 -0
- metadata +205 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a16fccf807f95dbd8a16bf90c03314fda9564a6d
|
4
|
+
data.tar.gz: 3f10e1d226527b8550e4950c03632d9d9a2060cb
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 05f9b3a9ac337b3b93bf6ad3f2df444bacf8982ca67fcc61d9c071783732a8a6e2e89d74d3944a7cb1fd18a3768ad830734ece62dcdfbe191deb677450943869
|
7
|
+
data.tar.gz: ce8a364831494992b183ba50af1b2cb922ca154c44b657b688bd69a62e7417c1c57a6e10d5bbb0368c89b47739901069d37d4cced164cfc80281f3caa5bea24c
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2017 OnePageCRM
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,178 @@
|
|
1
|
+
# Info
|
2
|
+
|
3
|
+
Rails db adapter for Impala layer over Kudu database.
|
4
|
+
|
5
|
+
# Usage
|
6
|
+
|
7
|
+
## Requirements
|
8
|
+
|
9
|
+
Active connection to Impala host and port 21000.
|
10
|
+
|
11
|
+
## Config database.yml
|
12
|
+
|
13
|
+
We need to config our yml like:
|
14
|
+
|
15
|
+
```
|
16
|
+
default: &default
|
17
|
+
adapter: kudu
|
18
|
+
host: 'localhost'
|
19
|
+
port: 21000
|
20
|
+
user: 'test'
|
21
|
+
password: 'test'
|
22
|
+
timeout: 5000
|
23
|
+
```
|
24
|
+
|
25
|
+
User and password not required.
|
26
|
+
|
27
|
+
## Supported migration functionality
|
28
|
+
|
29
|
+
With our adapter almost all migration functionality is supported, such as:
|
30
|
+
|
31
|
+
* create_table
|
32
|
+
* remove_table
|
33
|
+
* rename_table
|
34
|
+
* add_column
|
35
|
+
* remove_column
|
36
|
+
* rename_column
|
37
|
+
* and so on...
|
38
|
+
|
39
|
+
## Current limitations
|
40
|
+
|
41
|
+
* Creating any indexes (KUDU supports only primary key fields)
|
42
|
+
|
43
|
+
# Examples
|
44
|
+
|
45
|
+
Here is some examples...
|
46
|
+
|
47
|
+
## Create table
|
48
|
+
|
49
|
+
Because Kudu does not support AUTO INCREMENT INT fields we must ensure any primary key field is created as string field.
|
50
|
+
|
51
|
+
```
|
52
|
+
class CreateUsers < ActiveRecord::Migration[5.1]
|
53
|
+
def change
|
54
|
+
create_table :users, id: false do |t|
|
55
|
+
t.string :id, primary_key: true
|
56
|
+
t.string :account_id, null: false
|
57
|
+
t.string :name, null: false
|
58
|
+
t.string :email, null: false
|
59
|
+
t.string :company_id, null: false
|
60
|
+
t.string :company_name
|
61
|
+
t.timestamps
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
```
|
66
|
+
|
67
|
+
In example above we have only 1 primary key field and our model will work fully functionally when we using update(), delete() methods.
|
68
|
+
|
69
|
+
We can set additional primary key field, like
|
70
|
+
|
71
|
+
```
|
72
|
+
create_table :users, id: false do |t|
|
73
|
+
t.string :id, primary_key: true
|
74
|
+
t.string :account_id, primary_key: true
|
75
|
+
```
|
76
|
+
|
77
|
+
Here, we have two fields in primary key (id, account_id) and with KUDU table will be created with those 2 primary keys, but due to limitation of Rails will not be possible to use model delete(), update() methods.
|
78
|
+
|
79
|
+
## Add new column
|
80
|
+
|
81
|
+
Basic case:
|
82
|
+
|
83
|
+
```
|
84
|
+
class AddZipCodeToUsers < ActiveRecord::Migration[5.1]
|
85
|
+
def change
|
86
|
+
add_column :users, :zip_code, :string
|
87
|
+
end
|
88
|
+
end
|
89
|
+
```
|
90
|
+
|
91
|
+
In case of adding new primary key field, like:
|
92
|
+
|
93
|
+
```
|
94
|
+
class AddCompanyToUsers < ActiveRecord::Migration[5.1]
|
95
|
+
def up
|
96
|
+
add_column :users, :company_id, :string, primary_key: true
|
97
|
+
reload_table_data :users, :company_id, default: '#company-id'
|
98
|
+
end
|
99
|
+
def down
|
100
|
+
remove_column :users, :company_id
|
101
|
+
end
|
102
|
+
end
|
103
|
+
```
|
104
|
+
|
105
|
+
we will initialize specialized method at migration side, and basically this will happen:
|
106
|
+
|
107
|
+
* New table "table_name_redefined" will be created based on original table name (ex. users -> users_redefined) with included new field, new primary key field.
|
108
|
+
* Old table "users" will be renamed to "users_temp"
|
109
|
+
* Data will be copied from users_temp => users with additional new field and default value
|
110
|
+
* Old temporary table "users_temp" will be deleted
|
111
|
+
|
112
|
+
## Delete column
|
113
|
+
|
114
|
+
Basic case:
|
115
|
+
|
116
|
+
```
|
117
|
+
class RemoveZipCodeFromUsers < ActiveRecord::Migration[5.1]
|
118
|
+
def change
|
119
|
+
remove_column :users, :zip_code
|
120
|
+
end
|
121
|
+
end
|
122
|
+
```
|
123
|
+
|
124
|
+
In case of deleting primary key field (existing) procedure is same like for adding new column with primary key.
|
125
|
+
|
126
|
+
## Model associations
|
127
|
+
|
128
|
+
Model associations will work without foreign keys, like:
|
129
|
+
|
130
|
+
```
|
131
|
+
class CreateAccounts < ActiveRecord::Migration[5.1]
|
132
|
+
def change
|
133
|
+
create_table :accounts, id: false do |t|
|
134
|
+
t.string :id, primary_key: true
|
135
|
+
t.boolean :is_active, default: true
|
136
|
+
t.timestamps
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
class Account < ApplicationRecord
|
142
|
+
has_many :users, foreign_key: 'account_id'
|
143
|
+
end
|
144
|
+
```
|
145
|
+
|
146
|
+
and table Users with following construct...
|
147
|
+
|
148
|
+
```
|
149
|
+
class CreateUsers < ActiveRecord::Migration[5.1]
|
150
|
+
def change
|
151
|
+
create_table :users, id: false do |t|
|
152
|
+
t.string :id, primary_key: true
|
153
|
+
t.string :account_id, null: false
|
154
|
+
t.string :name, null: false
|
155
|
+
t.string :email, null: false
|
156
|
+
t.string :company_id, null: false
|
157
|
+
t.string :company_name
|
158
|
+
t.timestamps
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
class User < ApplicationRecord
|
164
|
+
belongs_to :account, primary_key: 'id'
|
165
|
+
end
|
166
|
+
```
|
167
|
+
|
168
|
+
This way we're able to do
|
169
|
+
|
170
|
+
```
|
171
|
+
Account.first.users => [#User, #User, ...]
|
172
|
+
```
|
173
|
+
|
174
|
+
or
|
175
|
+
|
176
|
+
```
|
177
|
+
User.first.account => #Account
|
178
|
+
```
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$LOAD_PATH.push File.expand_path('../lib', __FILE__)
|
4
|
+
require 'kudu_adapter/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = 'kudu_adapter'
|
8
|
+
s.version = KuduAdapter::VERSION
|
9
|
+
s.platform = Gem::Platform::RUBY
|
10
|
+
s.required_ruby_version = '>= 2.4.0'
|
11
|
+
s.authors = ['Paweł Smoliński', 'OnePageCRM']
|
12
|
+
s.licenses = ['MIT']
|
13
|
+
s.email = 'devteam@onepagecrm.com'
|
14
|
+
s.homepage = 'https://github.com/OnePageCRM/kudu_adapter'
|
15
|
+
s.summary = "ActiveRecord adapter for Cloudera's Kudu over Impala database"
|
16
|
+
s.description = "ActiveRecord adapter for Cloudera's Kudu over Impala database"
|
17
|
+
s.email = 'devteam@onepagecrm.com'
|
18
|
+
|
19
|
+
s.add_runtime_dependency('arel', ['~> 8.0'])
|
20
|
+
s.add_runtime_dependency('activemodel', ['~> 5.1.0'])
|
21
|
+
s.add_runtime_dependency('activerecord', ['~> 5.1.0'])
|
22
|
+
s.add_runtime_dependency('impala', ['~> 0.5.1'])
|
23
|
+
|
24
|
+
s.add_development_dependency('bundler')
|
25
|
+
s.add_development_dependency('rake')
|
26
|
+
s.add_development_dependency('rspec')
|
27
|
+
s.add_development_dependency('pry')
|
28
|
+
s.add_development_dependency('rubocop')
|
29
|
+
|
30
|
+
s.files = `git ls-files`.split("\n")
|
31
|
+
s.test_files = `git ls-files -- spec/*`.split("\n")
|
32
|
+
s.require_paths = ['lib']
|
33
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_record/connection_adapters/column'
|
4
|
+
|
5
|
+
# :nodoc:
|
6
|
+
module ActiveRecord
|
7
|
+
module ConnectionAdapters
|
8
|
+
module Kudu
|
9
|
+
# :nodoc:
|
10
|
+
class Column < ::ActiveRecord::ConnectionAdapters::Column
|
11
|
+
def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, comment = nil)
|
12
|
+
super(name, default, sql_type_metadata, null, table_name, nil, nil, comment: comment)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_record/result'
|
4
|
+
|
5
|
+
module ActiveRecord
|
6
|
+
module ConnectionAdapters
|
7
|
+
module Kudu
|
8
|
+
# :nodoc:
|
9
|
+
module DatabaseStatements
|
10
|
+
# :nodoc:
|
11
|
+
def exec_query(sql, _ = 'SQL', binds = [], prepare: false)
|
12
|
+
::Rails.logger.warn 'Prepared statements are not supported' if prepare
|
13
|
+
|
14
|
+
unless without_prepared_statement? binds
|
15
|
+
type_casted_binds(binds).each do |bind|
|
16
|
+
bind = quote(bind)
|
17
|
+
sql = sql.sub('?', bind.to_s)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
#::Rails.logger.info 'QUERY : ' + sql.to_s
|
21
|
+
result = connection.query sql
|
22
|
+
columns = result.first&.keys.to_a
|
23
|
+
rows = result.map { |row| row.fetch_values(*columns) }
|
24
|
+
::ActiveRecord::Result.new(columns.map(&:to_s), rows)
|
25
|
+
end
|
26
|
+
|
27
|
+
def exec_delete(sql, name, binds)
|
28
|
+
# We are not able to return number of affected rows so we will just say that there was some update
|
29
|
+
super
|
30
|
+
1
|
31
|
+
end
|
32
|
+
|
33
|
+
def exec_update(sql, name, binds)
|
34
|
+
# We are not able to return number of affected rows so we will just say that there was some update
|
35
|
+
super
|
36
|
+
1
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_support/core_ext/big_decimal/conversions'
|
4
|
+
require 'active_support/multibyte/chars'
|
5
|
+
|
6
|
+
module ActiveRecord
|
7
|
+
module ConnectionAdapters # :nodoc:
|
8
|
+
module Kudu
|
9
|
+
module Quoting
|
10
|
+
|
11
|
+
QUOTED_TRUE, QUOTED_FALSE = true.to_s, false.to_s
|
12
|
+
|
13
|
+
def quote_column_name(column_name)
|
14
|
+
column_name.to_s
|
15
|
+
end
|
16
|
+
|
17
|
+
def quote_table_name(table_name)
|
18
|
+
quote_column_name table_name
|
19
|
+
end
|
20
|
+
|
21
|
+
def quote_default_expression(value, column) # :nodoc:
|
22
|
+
if value.is_a?(Proc)
|
23
|
+
value.call
|
24
|
+
else
|
25
|
+
value = lookup_cast_type(column.sql_type).serialize(value)
|
26
|
+
# DOUBLE, FLOAT represented as 0.0 but KUDU supports only DEFAULT statement as 0
|
27
|
+
value = value.to_i if %w(DOUBLE FLOAT).include? column.sql_type
|
28
|
+
quote(value)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def quoted_true
|
33
|
+
QUOTED_TRUE
|
34
|
+
end
|
35
|
+
|
36
|
+
def unquoted_true
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
def quoted_false
|
41
|
+
QUOTED_FALSE
|
42
|
+
end
|
43
|
+
|
44
|
+
def unquoted_false
|
45
|
+
false
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'active_record/connection_adapters/abstract/schema_creation'
|
4
|
+
|
5
|
+
module ActiveRecord
|
6
|
+
module ConnectionAdapters
|
7
|
+
module Kudu
|
8
|
+
# :nodoc:
|
9
|
+
class SchemaCreation < ::ActiveRecord::ConnectionAdapters::AbstractAdapter::SchemaCreation
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def visit_AddColumnDefinition(obj)
|
14
|
+
"ADD COLUMNS (#{accept(obj.column)})"
|
15
|
+
end
|
16
|
+
|
17
|
+
def visit_ColumnDefinition(obj)
|
18
|
+
obj.sql_type = type_to_sql(obj.type, obj.options)
|
19
|
+
column_sql = "#{quote_column_name(obj.name)} #{obj.sql_type}".dup
|
20
|
+
add_column_options!(column_sql, column_options(obj))
|
21
|
+
end
|
22
|
+
|
23
|
+
# @param table_def [::ActiveRecord::ConnectionAdapters::Kudu::TableDefinition]
|
24
|
+
def visit_TableDefinition(table_def)
|
25
|
+
create_sql = "CREATE#{' EXTERNAL' if table_def.external} TABLE #{quote_table_name(table_def.name)} "
|
26
|
+
|
27
|
+
statements = table_def.columns.map { |col| accept col }
|
28
|
+
|
29
|
+
primary_keys = if table_def.primary_keys&.any?
|
30
|
+
table_def.primary_keys
|
31
|
+
else
|
32
|
+
table_def.columns.select { |col| col.options[:primary_key] }.map(&:name)
|
33
|
+
end
|
34
|
+
|
35
|
+
raise "Table #{table_def.name} does not have primary key(s) defined" if primary_keys.empty?
|
36
|
+
quoted_names = primary_keys.map { |pk| quote_column_name(pk) }
|
37
|
+
|
38
|
+
statements << "PRIMARY KEY (#{quoted_names.join(', ')})"
|
39
|
+
|
40
|
+
create_sql += "(#{statements.join(', ')})" if statements.present?
|
41
|
+
add_table_options!(create_sql, table_options(table_def))
|
42
|
+
|
43
|
+
# For managed Kudu tables partitioning must be defined
|
44
|
+
unless table_def.external
|
45
|
+
# If no partition columns will be provided, we will use all primary keys defined
|
46
|
+
partition_columns = table_def.partition_columns || primary_keys
|
47
|
+
if (partition_columns - table_def.columns.map(&:name)).any?
|
48
|
+
raise 'Non-existing columns have been selected as partition indicators'
|
49
|
+
end
|
50
|
+
|
51
|
+
partitions_count = table_def.partitions_count || 2
|
52
|
+
quoted_names = partition_columns.map { |pc| quote_column_name(pc) }
|
53
|
+
create_sql += " PARTITION BY HASH(#{quoted_names.join(', ')}) PARTITIONS #{partitions_count.to_i}"
|
54
|
+
# TODO: partitions range
|
55
|
+
end
|
56
|
+
|
57
|
+
create_sql + ' STORED AS KUDU'
|
58
|
+
end
|
59
|
+
|
60
|
+
def add_column_options!(sql, options)
|
61
|
+
# [NOT] NULL
|
62
|
+
if options[:primary_key]
|
63
|
+
sql += ' NOT NULL'
|
64
|
+
else
|
65
|
+
options[:null] = true if options[:null].nil?
|
66
|
+
sql += options[:null] ? ' NULL' : ' NOT NULL'
|
67
|
+
end
|
68
|
+
|
69
|
+
# Encodings:
|
70
|
+
# AUTO_ENCODING, PLAIN_ENCODING, RLE, DICT_ENCODING, BIT_SHUFFLE, PREFIX_ENCODING
|
71
|
+
sql += " ENCODING #{options[:encoding].to_s}" if options[:encoding]
|
72
|
+
|
73
|
+
# Compressions:
|
74
|
+
# LZ4, SNAPPY, and ZLIB
|
75
|
+
sql += " COMPRESSION #{options[:compression].to_s}" if options[:compression]
|
76
|
+
|
77
|
+
# Default values
|
78
|
+
sql += " DEFAULT #{quote_default_expression(options[:default], options[:column])}" unless options[:default].nil?
|
79
|
+
|
80
|
+
# Block size
|
81
|
+
sql += " BLOCK SIZE #{options[:block_size].to_i}" if options[:block_size]
|
82
|
+
|
83
|
+
sql
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|