kudu_adapter 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +4 -0
  3. data/.rubocop.yml +8 -0
  4. data/Gemfile +9 -0
  5. data/LICENSE.txt +20 -0
  6. data/README.md +178 -0
  7. data/kudu_adapter.gemspec +33 -0
  8. data/lib/active_record/connection_adapters/kudu/column.rb +17 -0
  9. data/lib/active_record/connection_adapters/kudu/database_statements.rb +41 -0
  10. data/lib/active_record/connection_adapters/kudu/quoting.rb +51 -0
  11. data/lib/active_record/connection_adapters/kudu/schema_creation.rb +89 -0
  12. data/lib/active_record/connection_adapters/kudu/schema_statements.rb +507 -0
  13. data/lib/active_record/connection_adapters/kudu/sql_type_metadata.rb +16 -0
  14. data/lib/active_record/connection_adapters/kudu/table_definition.rb +32 -0
  15. data/lib/active_record/connection_adapters/kudu/type/big_int.rb +22 -0
  16. data/lib/active_record/connection_adapters/kudu/type/boolean.rb +23 -0
  17. data/lib/active_record/connection_adapters/kudu/type/char.rb +17 -0
  18. data/lib/active_record/connection_adapters/kudu/type/date_time.rb +21 -0
  19. data/lib/active_record/connection_adapters/kudu/type/double.rb +17 -0
  20. data/lib/active_record/connection_adapters/kudu/type/float.rb +18 -0
  21. data/lib/active_record/connection_adapters/kudu/type/integer.rb +22 -0
  22. data/lib/active_record/connection_adapters/kudu/type/small_int.rb +22 -0
  23. data/lib/active_record/connection_adapters/kudu/type/string.rb +17 -0
  24. data/lib/active_record/connection_adapters/kudu/type/time.rb +30 -0
  25. data/lib/active_record/connection_adapters/kudu/type/tiny_int.rb +22 -0
  26. data/lib/active_record/connection_adapters/kudu_adapter.rb +173 -0
  27. data/lib/active_record/tasks/kudu_database_tasks.rb +29 -0
  28. data/lib/arel/visitors/kudu.rb +7 -0
  29. data/lib/kudu_adapter/bind_substitution.rb +15 -0
  30. data/lib/kudu_adapter/table_definition_extensions.rb +28 -0
  31. data/lib/kudu_adapter/version.rb +5 -0
  32. data/lib/kudu_adapter.rb +5 -0
  33. data/spec/spec_config.yaml.template +8 -0
  34. data/spec/spec_helper.rb +124 -0
  35. metadata +205 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a16fccf807f95dbd8a16bf90c03314fda9564a6d
4
+ data.tar.gz: 3f10e1d226527b8550e4950c03632d9d9a2060cb
5
+ SHA512:
6
+ metadata.gz: 05f9b3a9ac337b3b93bf6ad3f2df444bacf8982ca67fcc61d9c071783732a8a6e2e89d74d3944a7cb1fd18a3768ad830734ece62dcdfbe191deb677450943869
7
+ data.tar.gz: ce8a364831494992b183ba50af1b2cb922ca154c44b657b688bd69a62e7417c1c57a6e10d5bbb0368c89b47739901069d37d4cced164cfc80281f3caa5bea24c
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ syntax: glob
2
+ .idea/
3
+ Gemfile.lock
4
+ spec_config.yaml
data/.rubocop.yml ADDED
@@ -0,0 +1,8 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.4
3
+ Exclude:
4
+ - '**/schema_creation.rb'
5
+
6
+ Metrics/LineLength:
7
+ Enabled: true
8
+ Max: 120
data/Gemfile ADDED
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ source 'https://rubygems.org'
4
+
5
+ gemspec
6
+
7
+ group :test do
8
+ gem 'simplecov', require: false
9
+ end
data/LICENSE.txt ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2017 OnePageCRM
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,178 @@
1
+ # Info
2
+
3
+ Rails db adapter for Impala layer over Kudu database.
4
+
5
+ # Usage
6
+
7
+ ## Requirements
8
+
9
+ Active connection to Impala host and port 21000.
10
+
11
+ ## Config database.yml
12
+
13
+ We need to config our yml like:
14
+
15
+ ```
16
+ default: &default
17
+ adapter: kudu
18
+ host: 'localhost'
19
+ port: 21000
20
+ user: 'test'
21
+ password: 'test'
22
+ timeout: 5000
23
+ ```
24
+
25
+ User and password not required.
26
+
27
+ ## Supported migration functionality
28
+
29
+ With our adapter almost all migration functionality is supported, such as:
30
+
31
+ * create_table
32
+ * remove_table
33
+ * rename_table
34
+ * add_column
35
+ * remove_column
36
+ * rename_column
37
+ * and so on...
38
+
39
+ ## Current limitations
40
+
41
+ * Creating any indexes (KUDU supports only primary key fields)
42
+
43
+ # Examples
44
+
45
+ Here is some examples...
46
+
47
+ ## Create table
48
+
49
+ Because Kudu does not support AUTO INCREMENT INT fields we must ensure any primary key field is created as string field.
50
+
51
+ ```
52
+ class CreateUsers < ActiveRecord::Migration[5.1]
53
+ def change
54
+ create_table :users, id: false do |t|
55
+ t.string :id, primary_key: true
56
+ t.string :account_id, null: false
57
+ t.string :name, null: false
58
+ t.string :email, null: false
59
+ t.string :company_id, null: false
60
+ t.string :company_name
61
+ t.timestamps
62
+ end
63
+ end
64
+ end
65
+ ```
66
+
67
+ In example above we have only 1 primary key field and our model will work fully functionally when we using update(), delete() methods.
68
+
69
+ We can set additional primary key field, like
70
+
71
+ ```
72
+ create_table :users, id: false do |t|
73
+ t.string :id, primary_key: true
74
+ t.string :account_id, primary_key: true
75
+ ```
76
+
77
+ Here, we have two fields in primary key (id, account_id) and with KUDU table will be created with those 2 primary keys, but due to limitation of Rails will not be possible to use model delete(), update() methods.
78
+
79
+ ## Add new column
80
+
81
+ Basic case:
82
+
83
+ ```
84
+ class AddZipCodeToUsers < ActiveRecord::Migration[5.1]
85
+ def change
86
+ add_column :users, :zip_code, :string
87
+ end
88
+ end
89
+ ```
90
+
91
+ In case of adding new primary key field, like:
92
+
93
+ ```
94
+ class AddCompanyToUsers < ActiveRecord::Migration[5.1]
95
+ def up
96
+ add_column :users, :company_id, :string, primary_key: true
97
+ reload_table_data :users, :company_id, default: '#company-id'
98
+ end
99
+ def down
100
+ remove_column :users, :company_id
101
+ end
102
+ end
103
+ ```
104
+
105
+ we will initialize specialized method at migration side, and basically this will happen:
106
+
107
+ * New table "table_name_redefined" will be created based on original table name (ex. users -> users_redefined) with included new field, new primary key field.
108
+ * Old table "users" will be renamed to "users_temp"
109
+ * Data will be copied from users_temp => users with additional new field and default value
110
+ * Old temporary table "users_temp" will be deleted
111
+
112
+ ## Delete column
113
+
114
+ Basic case:
115
+
116
+ ```
117
+ class RemoveZipCodeFromUsers < ActiveRecord::Migration[5.1]
118
+ def change
119
+ remove_column :users, :zip_code
120
+ end
121
+ end
122
+ ```
123
+
124
+ In case of deleting primary key field (existing) procedure is same like for adding new column with primary key.
125
+
126
+ ## Model associations
127
+
128
+ Model associations will work without foreign keys, like:
129
+
130
+ ```
131
+ class CreateAccounts < ActiveRecord::Migration[5.1]
132
+ def change
133
+ create_table :accounts, id: false do |t|
134
+ t.string :id, primary_key: true
135
+ t.boolean :is_active, default: true
136
+ t.timestamps
137
+ end
138
+ end
139
+ end
140
+
141
+ class Account < ApplicationRecord
142
+ has_many :users, foreign_key: 'account_id'
143
+ end
144
+ ```
145
+
146
+ and table Users with following construct...
147
+
148
+ ```
149
+ class CreateUsers < ActiveRecord::Migration[5.1]
150
+ def change
151
+ create_table :users, id: false do |t|
152
+ t.string :id, primary_key: true
153
+ t.string :account_id, null: false
154
+ t.string :name, null: false
155
+ t.string :email, null: false
156
+ t.string :company_id, null: false
157
+ t.string :company_name
158
+ t.timestamps
159
+ end
160
+ end
161
+ end
162
+
163
+ class User < ApplicationRecord
164
+ belongs_to :account, primary_key: 'id'
165
+ end
166
+ ```
167
+
168
+ This way we're able to do
169
+
170
+ ```
171
+ Account.first.users => [#User, #User, ...]
172
+ ```
173
+
174
+ or
175
+
176
+ ```
177
+ User.first.account => #Account
178
+ ```
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ $LOAD_PATH.push File.expand_path('../lib', __FILE__)
4
+ require 'kudu_adapter/version'
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = 'kudu_adapter'
8
+ s.version = KuduAdapter::VERSION
9
+ s.platform = Gem::Platform::RUBY
10
+ s.required_ruby_version = '>= 2.4.0'
11
+ s.authors = ['Paweł Smoliński', 'OnePageCRM']
12
+ s.licenses = ['MIT']
13
+ s.email = 'devteam@onepagecrm.com'
14
+ s.homepage = 'https://github.com/OnePageCRM/kudu_adapter'
15
+ s.summary = "ActiveRecord adapter for Cloudera's Kudu over Impala database"
16
+ s.description = "ActiveRecord adapter for Cloudera's Kudu over Impala database"
17
+ s.email = 'devteam@onepagecrm.com'
18
+
19
+ s.add_runtime_dependency('arel', ['~> 8.0'])
20
+ s.add_runtime_dependency('activemodel', ['~> 5.1.0'])
21
+ s.add_runtime_dependency('activerecord', ['~> 5.1.0'])
22
+ s.add_runtime_dependency('impala', ['~> 0.5.1'])
23
+
24
+ s.add_development_dependency('bundler')
25
+ s.add_development_dependency('rake')
26
+ s.add_development_dependency('rspec')
27
+ s.add_development_dependency('pry')
28
+ s.add_development_dependency('rubocop')
29
+
30
+ s.files = `git ls-files`.split("\n")
31
+ s.test_files = `git ls-files -- spec/*`.split("\n")
32
+ s.require_paths = ['lib']
33
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_record/connection_adapters/column'
4
+
5
+ # :nodoc:
6
+ module ActiveRecord
7
+ module ConnectionAdapters
8
+ module Kudu
9
+ # :nodoc:
10
+ class Column < ::ActiveRecord::ConnectionAdapters::Column
11
+ def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, comment = nil)
12
+ super(name, default, sql_type_metadata, null, table_name, nil, nil, comment: comment)
13
+ end
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_record/result'
4
+
5
+ module ActiveRecord
6
+ module ConnectionAdapters
7
+ module Kudu
8
+ # :nodoc:
9
+ module DatabaseStatements
10
+ # :nodoc:
11
+ def exec_query(sql, _ = 'SQL', binds = [], prepare: false)
12
+ ::Rails.logger.warn 'Prepared statements are not supported' if prepare
13
+
14
+ unless without_prepared_statement? binds
15
+ type_casted_binds(binds).each do |bind|
16
+ bind = quote(bind)
17
+ sql = sql.sub('?', bind.to_s)
18
+ end
19
+ end
20
+ #::Rails.logger.info 'QUERY : ' + sql.to_s
21
+ result = connection.query sql
22
+ columns = result.first&.keys.to_a
23
+ rows = result.map { |row| row.fetch_values(*columns) }
24
+ ::ActiveRecord::Result.new(columns.map(&:to_s), rows)
25
+ end
26
+
27
+ def exec_delete(sql, name, binds)
28
+ # We are not able to return number of affected rows so we will just say that there was some update
29
+ super
30
+ 1
31
+ end
32
+
33
+ def exec_update(sql, name, binds)
34
+ # We are not able to return number of affected rows so we will just say that there was some update
35
+ super
36
+ 1
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,51 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_support/core_ext/big_decimal/conversions'
4
+ require 'active_support/multibyte/chars'
5
+
6
+ module ActiveRecord
7
+ module ConnectionAdapters # :nodoc:
8
+ module Kudu
9
+ module Quoting
10
+
11
+ QUOTED_TRUE, QUOTED_FALSE = true.to_s, false.to_s
12
+
13
+ def quote_column_name(column_name)
14
+ column_name.to_s
15
+ end
16
+
17
+ def quote_table_name(table_name)
18
+ quote_column_name table_name
19
+ end
20
+
21
+ def quote_default_expression(value, column) # :nodoc:
22
+ if value.is_a?(Proc)
23
+ value.call
24
+ else
25
+ value = lookup_cast_type(column.sql_type).serialize(value)
26
+ # DOUBLE, FLOAT represented as 0.0 but KUDU supports only DEFAULT statement as 0
27
+ value = value.to_i if %w(DOUBLE FLOAT).include? column.sql_type
28
+ quote(value)
29
+ end
30
+ end
31
+
32
+ def quoted_true
33
+ QUOTED_TRUE
34
+ end
35
+
36
+ def unquoted_true
37
+ true
38
+ end
39
+
40
+ def quoted_false
41
+ QUOTED_FALSE
42
+ end
43
+
44
+ def unquoted_false
45
+ false
46
+ end
47
+
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_record/connection_adapters/abstract/schema_creation'
4
+
5
+ module ActiveRecord
6
+ module ConnectionAdapters
7
+ module Kudu
8
+ # :nodoc:
9
+ class SchemaCreation < ::ActiveRecord::ConnectionAdapters::AbstractAdapter::SchemaCreation
10
+
11
+ private
12
+
13
+ def visit_AddColumnDefinition(obj)
14
+ "ADD COLUMNS (#{accept(obj.column)})"
15
+ end
16
+
17
+ def visit_ColumnDefinition(obj)
18
+ obj.sql_type = type_to_sql(obj.type, obj.options)
19
+ column_sql = "#{quote_column_name(obj.name)} #{obj.sql_type}".dup
20
+ add_column_options!(column_sql, column_options(obj))
21
+ end
22
+
23
+ # @param table_def [::ActiveRecord::ConnectionAdapters::Kudu::TableDefinition]
24
+ def visit_TableDefinition(table_def)
25
+ create_sql = "CREATE#{' EXTERNAL' if table_def.external} TABLE #{quote_table_name(table_def.name)} "
26
+
27
+ statements = table_def.columns.map { |col| accept col }
28
+
29
+ primary_keys = if table_def.primary_keys&.any?
30
+ table_def.primary_keys
31
+ else
32
+ table_def.columns.select { |col| col.options[:primary_key] }.map(&:name)
33
+ end
34
+
35
+ raise "Table #{table_def.name} does not have primary key(s) defined" if primary_keys.empty?
36
+ quoted_names = primary_keys.map { |pk| quote_column_name(pk) }
37
+
38
+ statements << "PRIMARY KEY (#{quoted_names.join(', ')})"
39
+
40
+ create_sql += "(#{statements.join(', ')})" if statements.present?
41
+ add_table_options!(create_sql, table_options(table_def))
42
+
43
+ # For managed Kudu tables partitioning must be defined
44
+ unless table_def.external
45
+ # If no partition columns will be provided, we will use all primary keys defined
46
+ partition_columns = table_def.partition_columns || primary_keys
47
+ if (partition_columns - table_def.columns.map(&:name)).any?
48
+ raise 'Non-existing columns have been selected as partition indicators'
49
+ end
50
+
51
+ partitions_count = table_def.partitions_count || 2
52
+ quoted_names = partition_columns.map { |pc| quote_column_name(pc) }
53
+ create_sql += " PARTITION BY HASH(#{quoted_names.join(', ')}) PARTITIONS #{partitions_count.to_i}"
54
+ # TODO: partitions range
55
+ end
56
+
57
+ create_sql + ' STORED AS KUDU'
58
+ end
59
+
60
+ def add_column_options!(sql, options)
61
+ # [NOT] NULL
62
+ if options[:primary_key]
63
+ sql += ' NOT NULL'
64
+ else
65
+ options[:null] = true if options[:null].nil?
66
+ sql += options[:null] ? ' NULL' : ' NOT NULL'
67
+ end
68
+
69
+ # Encodings:
70
+ # AUTO_ENCODING, PLAIN_ENCODING, RLE, DICT_ENCODING, BIT_SHUFFLE, PREFIX_ENCODING
71
+ sql += " ENCODING #{options[:encoding].to_s}" if options[:encoding]
72
+
73
+ # Compressions:
74
+ # LZ4, SNAPPY, and ZLIB
75
+ sql += " COMPRESSION #{options[:compression].to_s}" if options[:compression]
76
+
77
+ # Default values
78
+ sql += " DEFAULT #{quote_default_expression(options[:default], options[:column])}" unless options[:default].nil?
79
+
80
+ # Block size
81
+ sql += " BLOCK SIZE #{options[:block_size].to_i}" if options[:block_size]
82
+
83
+ sql
84
+ end
85
+
86
+ end
87
+ end
88
+ end
89
+ end