clickhouse-activerecord 1.0.3 → 1.0.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a41725d372314184b6799381965f0fb299a42d27b5c714bd04ad768ab0f87a11
4
- data.tar.gz: 12e892096d7d76f039f484e58b045c9f680ee95297790f485f8b19d9ecca7412
3
+ metadata.gz: 2217c1539b22307398fbc9c01bb8b521300ca23a7d03c2d32a56a9abdd3c5546
4
+ data.tar.gz: 8aa60f1a78db38df4120b0b405ddaf9466da70e68bf382f58945b29d5e130b7b
5
5
  SHA512:
6
- metadata.gz: 35afbee04dec61f61685ec9edd70a305df94fa2198933c9271f3221258f069be06a684dfd10a0a203798712ab1113f91b4f773d8c1f85f1455e4511b6437d6be
7
- data.tar.gz: 0b9adb5b3da648ab1ca1b44352b58fc91d7d09b0c29cbc816e23faa39b060ca3e7a8f35e67eb87e92332cae513592f19378fcf993c098684463ffbcd817d64e2
6
+ metadata.gz: d1e48e302475eff9ca738b35fc56b58a7a73f7d6b6cf5ae6632a7ee95fbc9432734ba18c53f7756d833ebd3748368fd73f7c29e04a0972f3141d00b5acd262cd
7
+ data.tar.gz: acedbb7298cd044769a5a1deaaae44672bdbf1affca5d2218e8b7209731df505434ed4f69bb23881e0d4f231e9e6699ac621ab0b4ac19f86475dd6c6515b3e18
@@ -0,0 +1,117 @@
1
+ <?xml version="1.0"?>
2
+ <clickhouse>
3
+
4
+ <http_port>8123</http_port>
5
+ <interserver_http_port>9009</interserver_http_port>
6
+ <interserver_http_host>clickhouse1</interserver_http_host>
7
+
8
+ <users_config>users.xml</users_config>
9
+ <default_profile>default</default_profile>
10
+ <default_database>default</default_database>
11
+
12
+ <mark_cache_size>5368709120</mark_cache_size>
13
+
14
+ <path>/var/lib/clickhouse/</path>
15
+ <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
16
+ <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
17
+ <access_control_path>/var/lib/clickhouse/access/</access_control_path>
18
+ <keep_alive_timeout>3</keep_alive_timeout>
19
+
20
+ <logger>
21
+ <level>debug</level>
22
+ <log>/var/log/clickhouse-server/clickhouse-server.log</log>
23
+ <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
24
+ <size>1000M</size>
25
+ <count>10</count>
26
+ <console>1</console>
27
+ </logger>
28
+
29
+ <remote_servers>
30
+ <test_cluster>
31
+ <shard>
32
+ <replica>
33
+ <host>clickhouse1</host>
34
+ <port>9000</port>
35
+ </replica>
36
+ <replica>
37
+ <host>clickhouse2</host>
38
+ <port>9000</port>
39
+ </replica>
40
+ </shard>
41
+ </test_cluster>
42
+ </remote_servers>
43
+
44
+ <keeper_server>
45
+ <tcp_port>9181</tcp_port>
46
+ <server_id>1</server_id>
47
+ <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
48
+ <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
49
+
50
+ <coordination_settings>
51
+ <operation_timeout_ms>10000</operation_timeout_ms>
52
+ <session_timeout_ms>30000</session_timeout_ms>
53
+ <raft_logs_level>trace</raft_logs_level>
54
+ <rotate_log_storage_interval>10000</rotate_log_storage_interval>
55
+ </coordination_settings>
56
+
57
+ <raft_configuration>
58
+ <server>
59
+ <id>1</id>
60
+ <hostname>clickhouse1</hostname>
61
+ <port>9000</port>
62
+ </server>
63
+ <server>
64
+ <id>2</id>
65
+ <hostname>clickhouse2</hostname>
66
+ <port>9000</port>
67
+ </server>
68
+ </raft_configuration>
69
+ </keeper_server>
70
+
71
+ <zookeeper>
72
+ <node>
73
+ <host>clickhouse1</host>
74
+ <port>9181</port>
75
+ </node>
76
+ <node>
77
+ <host>clickhouse2</host>
78
+ <port>9181</port>
79
+ </node>
80
+ </zookeeper>
81
+
82
+ <macros>
83
+ <cluster>test_cluster</cluster>
84
+ <replica>clickhouse1</replica>
85
+ <shard>1</shard>
86
+ </macros>
87
+
88
+ <distributed_ddl>
89
+ <path>/clickhouse/test_cluster/task_queue/ddl</path>
90
+ </distributed_ddl>
91
+
92
+ <query_log>
93
+ <database>system</database>
94
+ <table>query_log</table>
95
+ <partition_by>toYYYYMM(event_date)</partition_by>
96
+ <flush_interval_milliseconds>1000</flush_interval_milliseconds>
97
+ </query_log>
98
+
99
+ <http_options_response>
100
+ <header>
101
+ <name>Access-Control-Allow-Origin</name>
102
+ <value>*</value>
103
+ </header>
104
+ <header>
105
+ <name>Access-Control-Allow-Headers</name>
106
+ <value>accept, origin, x-requested-with, content-type, authorization</value>
107
+ </header>
108
+ <header>
109
+ <name>Access-Control-Allow-Methods</name>
110
+ <value>POST, GET, OPTIONS</value>
111
+ </header>
112
+ <header>
113
+ <name>Access-Control-Max-Age</name>
114
+ <value>86400</value>
115
+ </header>
116
+ </http_options_response>
117
+ </clickhouse>
@@ -0,0 +1,117 @@
1
+ <?xml version="1.0"?>
2
+ <clickhouse>
3
+
4
+ <http_port>8123</http_port>
5
+ <interserver_http_port>9009</interserver_http_port>
6
+ <interserver_http_host>clickhouse2</interserver_http_host>
7
+
8
+ <users_config>users.xml</users_config>
9
+ <default_profile>default</default_profile>
10
+ <default_database>default</default_database>
11
+
12
+ <mark_cache_size>5368709120</mark_cache_size>
13
+
14
+ <path>/var/lib/clickhouse/</path>
15
+ <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
16
+ <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
17
+ <access_control_path>/var/lib/clickhouse/access/</access_control_path>
18
+ <keep_alive_timeout>3</keep_alive_timeout>
19
+
20
+ <logger>
21
+ <level>debug</level>
22
+ <log>/var/log/clickhouse-server/clickhouse-server.log</log>
23
+ <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
24
+ <size>1000M</size>
25
+ <count>10</count>
26
+ <console>1</console>
27
+ </logger>
28
+
29
+ <remote_servers>
30
+ <test_cluster>
31
+ <shard>
32
+ <replica>
33
+ <host>clickhouse1</host>
34
+ <port>9000</port>
35
+ </replica>
36
+ <replica>
37
+ <host>clickhouse2</host>
38
+ <port>9000</port>
39
+ </replica>
40
+ </shard>
41
+ </test_cluster>
42
+ </remote_servers>
43
+
44
+ <keeper_server>
45
+ <tcp_port>9181</tcp_port>
46
+ <server_id>2</server_id>
47
+ <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
48
+ <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
49
+
50
+ <coordination_settings>
51
+ <operation_timeout_ms>10000</operation_timeout_ms>
52
+ <session_timeout_ms>30000</session_timeout_ms>
53
+ <raft_logs_level>trace</raft_logs_level>
54
+ <rotate_log_storage_interval>10000</rotate_log_storage_interval>
55
+ </coordination_settings>
56
+
57
+ <raft_configuration>
58
+ <server>
59
+ <id>1</id>
60
+ <hostname>clickhouse1</hostname>
61
+ <port>9000</port>
62
+ </server>
63
+ <server>
64
+ <id>2</id>
65
+ <hostname>clickhouse2</hostname>
66
+ <port>9000</port>
67
+ </server>
68
+ </raft_configuration>
69
+ </keeper_server>
70
+
71
+ <zookeeper>
72
+ <node>
73
+ <host>clickhouse1</host>
74
+ <port>9181</port>
75
+ </node>
76
+ <node>
77
+ <host>clickhouse2</host>
78
+ <port>9181</port>
79
+ </node>
80
+ </zookeeper>
81
+
82
+ <macros>
83
+ <cluster>test_cluster</cluster>
84
+ <replica>clickhouse2</replica>
85
+ <shard>1</shard>
86
+ </macros>
87
+
88
+ <distributed_ddl>
89
+ <path>/clickhouse/test_cluster/task_queue/ddl</path>
90
+ </distributed_ddl>
91
+
92
+ <query_log>
93
+ <database>system</database>
94
+ <table>query_log</table>
95
+ <partition_by>toYYYYMM(event_date)</partition_by>
96
+ <flush_interval_milliseconds>1000</flush_interval_milliseconds>
97
+ </query_log>
98
+
99
+ <http_options_response>
100
+ <header>
101
+ <name>Access-Control-Allow-Origin</name>
102
+ <value>*</value>
103
+ </header>
104
+ <header>
105
+ <name>Access-Control-Allow-Headers</name>
106
+ <value>accept, origin, x-requested-with, content-type, authorization</value>
107
+ </header>
108
+ <header>
109
+ <name>Access-Control-Allow-Methods</name>
110
+ <value>POST, GET, OPTIONS</value>
111
+ </header>
112
+ <header>
113
+ <name>Access-Control-Max-Age</name>
114
+ <value>86400</value>
115
+ </header>
116
+ </http_options_response>
117
+ </clickhouse>
@@ -0,0 +1,54 @@
1
+ <?xml version="1.0"?>
2
+ <clickhouse>
3
+
4
+ <http_port>8123</http_port>
5
+ <tcp_port>9000</tcp_port>
6
+
7
+ <users_config>users.xml</users_config>
8
+ <default_profile>default</default_profile>
9
+ <default_database>default</default_database>
10
+
11
+ <mark_cache_size>5368709120</mark_cache_size>
12
+
13
+ <path>/var/lib/clickhouse/</path>
14
+ <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
15
+ <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
16
+ <access_control_path>/var/lib/clickhouse/access/</access_control_path>
17
+ <keep_alive_timeout>3</keep_alive_timeout>
18
+
19
+ <logger>
20
+ <level>debug</level>
21
+ <log>/var/log/clickhouse-server/clickhouse-server.log</log>
22
+ <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
23
+ <size>1000M</size>
24
+ <count>10</count>
25
+ <console>1</console>
26
+ </logger>
27
+
28
+ <query_log>
29
+ <database>system</database>
30
+ <table>query_log</table>
31
+ <partition_by>toYYYYMM(event_date)</partition_by>
32
+ <flush_interval_milliseconds>1000</flush_interval_milliseconds>
33
+ </query_log>
34
+
35
+ <http_options_response>
36
+ <header>
37
+ <name>Access-Control-Allow-Origin</name>
38
+ <value>*</value>
39
+ </header>
40
+ <header>
41
+ <name>Access-Control-Allow-Headers</name>
42
+ <value>accept, origin, x-requested-with, content-type, authorization</value>
43
+ </header>
44
+ <header>
45
+ <name>Access-Control-Allow-Methods</name>
46
+ <value>POST, GET, OPTIONS</value>
47
+ </header>
48
+ <header>
49
+ <name>Access-Control-Max-Age</name>
50
+ <value>86400</value>
51
+ </header>
52
+ </http_options_response>
53
+
54
+ </clickhouse>
@@ -0,0 +1,34 @@
1
+ <?xml version="1.0"?>
2
+ <clickhouse>
3
+
4
+ <profiles>
5
+ <default>
6
+ <load_balancing>random</load_balancing>
7
+ </default>
8
+ </profiles>
9
+
10
+ <users>
11
+ <default>
12
+ <password></password>
13
+ <networks>
14
+ <ip>::/0</ip>
15
+ </networks>
16
+ <profile>default</profile>
17
+ <quota>default</quota>
18
+ <access_management>1</access_management>
19
+ </default>
20
+ </users>
21
+
22
+ <quotas>
23
+ <default>
24
+ <interval>
25
+ <duration>3600</duration>
26
+ <queries>0</queries>
27
+ <errors>0</errors>
28
+ <result_rows>0</result_rows>
29
+ <read_rows>0</read_rows>
30
+ <execution_time>0</execution_time>
31
+ </interval>
32
+ </default>
33
+ </quotas>
34
+ </clickhouse>
@@ -0,0 +1,41 @@
1
+ version: '3.5'
2
+
3
+ services:
4
+ clickhouse1:
5
+ image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
6
+ ulimits:
7
+ nofile:
8
+ soft: 262144
9
+ hard: 262144
10
+ hostname: clickhouse1
11
+ container_name: clickhouse-activerecord-clickhouse-server-1
12
+ ports:
13
+ - '8124:8123'
14
+ - '9001:9000'
15
+ volumes:
16
+ - './clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml'
17
+ - './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
18
+
19
+ clickhouse2:
20
+ image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
21
+ ulimits:
22
+ nofile:
23
+ soft: 262144
24
+ hard: 262144
25
+ hostname: clickhouse2
26
+ container_name: clickhouse-activerecord-clickhouse-server-2
27
+ ports:
28
+ - '8125:8123'
29
+ volumes:
30
+ - './clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml'
31
+ - './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
32
+
33
+ # Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests
34
+ nginx:
35
+ image: 'nginx:1.23.1-alpine'
36
+ hostname: nginx
37
+ ports:
38
+ - '28123:8123'
39
+ volumes:
40
+ - './nginx/local.conf:/etc/nginx/conf.d/local.conf'
41
+ container_name: clickhouse-activerecord-nginx
@@ -0,0 +1,14 @@
1
+ version: '3.8'
2
+ services:
3
+ clickhouse:
4
+ image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
5
+ container_name: 'clickhouse-activerecord-clickhouse-server'
6
+ ports:
7
+ - '18123:8123'
8
+ ulimits:
9
+ nofile:
10
+ soft: 262144
11
+ hard: 262144
12
+ volumes:
13
+ - './clickhouse/single/config.xml:/etc/clickhouse-server/config.xml'
14
+ - './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
@@ -0,0 +1,12 @@
1
+ upstream clickhouse_cluster {
2
+ server clickhouse1:8123;
3
+ server clickhouse2:8123;
4
+ }
5
+
6
+ server {
7
+ listen 8123;
8
+ client_max_body_size 100M;
9
+ location / {
10
+ proxy_pass http://clickhouse_cluster;
11
+ }
12
+ }
@@ -0,0 +1,77 @@
1
+ name: Testing
2
+
3
+ on:
4
+ push:
5
+ branches: [ "master" ]
6
+ pull_request:
7
+ branches: [ "master" ]
8
+
9
+ jobs:
10
+ tests_single:
11
+ name: Testing single server
12
+ runs-on: ubuntu-latest
13
+
14
+ env:
15
+ CLICKHOUSE_PORT: 18123
16
+ CLICKHOUSE_DATABASE: default
17
+
18
+ strategy:
19
+ fail-fast: true
20
+ max-parallel: 1
21
+ matrix:
22
+ ruby-version: [ '2.7', '3.0', '3.2' ]
23
+ clickhouse: [ '22.1' ]
24
+
25
+ steps:
26
+ - uses: actions/checkout@v4
27
+
28
+ - name: Start ClickHouse ${{ matrix.clickhouse }}
29
+ uses: isbang/compose-action@v1.5.1
30
+ env:
31
+ CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
32
+ with:
33
+ compose-file: '.docker/docker-compose.yml'
34
+ down-flags: '--volumes'
35
+
36
+ - name: Set up Ruby ${{ matrix.ruby-version }}
37
+ uses: ruby/setup-ruby@v1
38
+ with:
39
+ ruby-version: ${{ matrix.ruby-version }}
40
+ bundler-cache: true
41
+
42
+ - run: bundle exec rspec spec/single
43
+
44
+ tests_cluster:
45
+ name: Testing cluster server
46
+ runs-on: ubuntu-latest
47
+
48
+ env:
49
+ CLICKHOUSE_PORT: 28123
50
+ CLICKHOUSE_DATABASE: default
51
+ CLICKHOUSE_CLUSTER: test_cluster
52
+
53
+ strategy:
54
+ fail-fast: true
55
+ max-parallel: 1
56
+ matrix:
57
+ ruby-version: [ '2.7', '3.0', '3.2' ]
58
+ clickhouse: [ '22.1' ]
59
+
60
+ steps:
61
+ - uses: actions/checkout@v4
62
+
63
+ - name: Start ClickHouse Cluster ${{ matrix.clickhouse }}
64
+ uses: isbang/compose-action@v1.5.1
65
+ env:
66
+ CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
67
+ with:
68
+ compose-file: '.docker/docker-compose.cluster.yml'
69
+ down-flags: '--volumes'
70
+
71
+ - name: Set up Ruby ${{ matrix.ruby-version }}
72
+ uses: ruby/setup-ruby@v1
73
+ with:
74
+ ruby-version: ${{ matrix.ruby-version }}
75
+ bundler-cache: true
76
+
77
+ - run: bundle exec rspec spec/cluster
data/CHANGELOG.md CHANGED
@@ -1,3 +1,31 @@
1
+ ### Version 1.0.5 (Mar 14, 2024)
2
+
3
+ * GitHub workflows
4
+ * Fix injection internal and schema classes for rails 7
5
+ * Add support for binary string by [@PauloMiranda98](https://github.com/PauloMiranda98) in (#116)
6
+
7
+ ### Version 1.0.4 (Feb 2, 2024)
8
+
9
+ * Use ILIKE for `model.arel_table[:column]#matches` by [@stympy](https://github.com/stympy) in (#115)
10
+ * Fixed `insert_all` for array column (#71)
11
+ * Register Bool and UUID in type map by [@lukinski](https://github.com/lukinski) in (#110)
12
+ * Refactoring `final` method
13
+ * Support update & delete for clickhouse from version 23.3 and newer (#93)
14
+
15
+ ### Version 1.0.0 (Nov 29, 2023)
16
+
17
+ * Full support Rails 7.1+
18
+ * Full support primary or multiple databases
19
+
20
+ ### Version 0.6.0 (Oct 19, 2023)
21
+
22
+ * Added `Bool` column type instead `Uint8` (#78). Supports ClickHouse 22+ database only
23
+ * Added `final` method (#81) (The `ar_internal_metadata` table needs to be deleted after a gem update)
24
+ * Added `settings` method (#82)
25
+ * Fixed convert aggregation type (#92)
26
+ * Fixed raise error not database exist (#91)
27
+ * Fixed internal metadata update (#84)
28
+
1
29
  ### Version 0.5.10 (Jun 22, 2022)
2
30
 
3
31
  * Fixes to create_table method (#70)
@@ -6,6 +6,8 @@ module ActiveRecord
6
6
  module ConnectionAdapters
7
7
  module Clickhouse
8
8
  module SchemaStatements
9
+ DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
10
+
9
11
  def execute(sql, name = nil, settings: {})
10
12
  do_execute(sql, name, settings: settings)
11
13
  end
@@ -33,13 +35,20 @@ module ActiveRecord
33
35
  # @link https://clickhouse.com/docs/en/sql-reference/statements/alter/update
34
36
  def exec_update(_sql, _name = nil, _binds = [])
35
37
  do_execute(_sql, _name, format: nil)
36
- true
38
+ 0
37
39
  end
38
40
 
39
41
  # @link https://clickhouse.com/docs/en/sql-reference/statements/delete
40
42
  def exec_delete(_sql, _name = nil, _binds = [])
41
- do_execute(_sql, _name, format: nil)
42
- true
43
+ log(_sql, "#{adapter_name} #{_name}") do
44
+ res = request(_sql)
45
+ begin
46
+ data = JSON.parse(res.header['x-clickhouse-summary'])
47
+ data['result_rows'].to_i
48
+ rescue JSONError
49
+ 0
50
+ end
51
+ end
43
52
  end
44
53
 
45
54
  def tables(name = nil)
@@ -64,19 +73,15 @@ module ActiveRecord
64
73
 
65
74
  def do_system_execute(sql, name = nil)
66
75
  log_with_debug(sql, "#{adapter_name} #{name}") do
67
- res = @connection.post("/?#{@config.to_param}", "#{sql} FORMAT JSONCompact", 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
68
-
69
- process_response(res)
76
+ res = request(sql, DEFAULT_RESPONSE_FORMAT)
77
+ process_response(res, DEFAULT_RESPONSE_FORMAT)
70
78
  end
71
79
  end
72
80
 
73
- def do_execute(sql, name = nil, format: 'JSONCompact', settings: {})
81
+ def do_execute(sql, name = nil, format: DEFAULT_RESPONSE_FORMAT, settings: {})
74
82
  log(sql, "#{adapter_name} #{name}") do
75
- formatted_sql = apply_format(sql, format)
76
- request_params = @config || {}
77
- res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
78
-
79
- process_response(res)
83
+ res = request(sql, format, settings)
84
+ process_response(res, format)
80
85
  end
81
86
  end
82
87
 
@@ -96,23 +101,44 @@ module ActiveRecord
96
101
  if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
97
102
  raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
98
103
  end
99
- do_execute(insert_versions_sql(inserting), nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
104
+ do_execute(insert_versions_sql(inserting), nil, format: nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
105
+ end
106
+ end
107
+
108
+ # Fix insert_all method
109
+ # https://github.com/PNixx/clickhouse-activerecord/issues/71#issuecomment-1923244983
110
+ def with_yaml_fallback(value) # :nodoc:
111
+ if value.is_a?(Array)
112
+ value
113
+ else
114
+ super
100
115
  end
101
116
  end
102
117
 
103
118
  private
104
119
 
120
+ # Make HTTP request to ClickHouse server
121
+ # @param [String] sql
122
+ # @param [String, nil] format
123
+ # @param [Hash] settings
124
+ # @return [Net::HTTPResponse]
125
+ def request(sql, format = nil, settings = {})
126
+ formatted_sql = apply_format(sql, format)
127
+ request_params = @connection_config || {}
128
+ @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
129
+ end
130
+
105
131
  def apply_format(sql, format)
106
132
  format ? "#{sql} FORMAT #{format}" : sql
107
133
  end
108
134
 
109
- def process_response(res)
135
+ def process_response(res, format)
110
136
  case res.code.to_i
111
137
  when 200
112
138
  if res.body.to_s.include?("DB::Exception")
113
139
  raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}"
114
140
  else
115
- res.body.presence && JSON.parse(res.body)
141
+ format_body_response(res.body, format)
116
142
  end
117
143
  else
118
144
  case res.body
@@ -158,8 +184,7 @@ module ActiveRecord
158
184
 
159
185
  return data unless data.empty?
160
186
 
161
- raise ActiveRecord::StatementInvalid,
162
- "Could not find table '#{table_name}'"
187
+ raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'"
163
188
  end
164
189
  alias column_definitions table_structure
165
190
 
@@ -193,6 +218,40 @@ module ActiveRecord
193
218
  def has_default_function?(default_value, default) # :nodoc:
194
219
  !default_value && (%r{\w+\(.*\)} === default)
195
220
  end
221
+
222
+ def format_body_response(body, format)
223
+ return body if body.blank?
224
+
225
+ case format
226
+ when 'JSONCompact'
227
+ format_from_json_compact(body)
228
+ when 'JSONCompactEachRowWithNamesAndTypes'
229
+ format_from_json_compact_each_row_with_names_and_types(body)
230
+ else
231
+ body
232
+ end
233
+ end
234
+
235
+ def format_from_json_compact(body)
236
+ JSON.parse(body)
237
+ end
238
+
239
+ def format_from_json_compact_each_row_with_names_and_types(body)
240
+ rows = body.split("\n").map { |row| JSON.parse(row) }
241
+ names, types, *data = rows
242
+
243
+ meta = names.zip(types).map do |name, type|
244
+ {
245
+ 'name' => name,
246
+ 'type' => type
247
+ }
248
+ end
249
+
250
+ {
251
+ 'meta' => meta,
252
+ 'data' => data
253
+ }
254
+ end
196
255
  end
197
256
  end
198
257
  end
@@ -45,7 +45,7 @@ module ActiveRecord
45
45
  raise ArgumentError, 'No database specified. Missing argument: database.'
46
46
  end
47
47
 
48
- ConnectionAdapters::ClickhouseAdapter.new(logger, connection, { user: config[:username], password: config[:password], database: database }.compact, config)
48
+ ConnectionAdapters::ClickhouseAdapter.new(logger, connection, config)
49
49
  end
50
50
  end
51
51
  end
@@ -73,10 +73,11 @@ module ActiveRecord
73
73
  def is_view=(value)
74
74
  @is_view = value
75
75
  end
76
- #
77
- # def arel_table # :nodoc:
78
- # @arel_table ||= Arel::Table.new(table_name, type_caster: type_caster)
79
- # end
76
+
77
+ def _delete_record(constraints)
78
+ raise ActiveRecord::ActiveRecordError.new('Deleting a row is not possible without a primary key') unless self.primary_key
79
+ super
80
+ end
80
81
  end
81
82
  end
82
83
 
@@ -120,12 +121,12 @@ module ActiveRecord
120
121
  include Clickhouse::SchemaStatements
121
122
 
122
123
  # Initializes and connects a Clickhouse adapter.
123
- def initialize(logger, connection_parameters, config, full_config)
124
+ def initialize(logger, connection_parameters, config)
124
125
  super(nil, logger)
125
126
  @connection_parameters = connection_parameters
127
+ @connection_config = { user: config[:username], password: config[:password], database: config[:database] }.compact
128
+ @debug = config[:debug] || false
126
129
  @config = config
127
- @debug = full_config[:debug] || false
128
- @full_config = full_config
129
130
 
130
131
  @prepared_statements = false
131
132
 
@@ -133,7 +134,7 @@ module ActiveRecord
133
134
  end
134
135
 
135
136
  def migrations_paths
136
- @full_config[:migrations_paths] || 'db/migrate_clickhouse'
137
+ @config[:migrations_paths] || 'db/migrate_clickhouse'
137
138
  end
138
139
 
139
140
  def arel_visitor # :nodoc:
@@ -266,8 +267,8 @@ module ActiveRecord
266
267
  def create_database(name)
267
268
  sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
268
269
  log_with_debug(sql, adapter_name) do
269
- res = @connection.post("/?#{@config.except(:database).to_param}", sql)
270
- process_response(res)
270
+ res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
271
+ process_response(res, DEFAULT_RESPONSE_FORMAT)
271
272
  end
272
273
  end
273
274
 
@@ -302,7 +303,7 @@ module ActiveRecord
302
303
  raise 'Set a cluster' unless cluster
303
304
 
304
305
  distributed_options =
305
- "Distributed(#{cluster}, #{@config[:database]}, #{table_name}, #{sharding_key})"
306
+ "Distributed(#{cluster}, #{@connection_config[:database]}, #{table_name}, #{sharding_key})"
306
307
  create_table(distributed_table_name, **options.merge(options: distributed_options), &block)
307
308
  end
308
309
  end
@@ -311,8 +312,8 @@ module ActiveRecord
311
312
  def drop_database(name) #:nodoc:
312
313
  sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
313
314
  log_with_debug(sql, adapter_name) do
314
- res = @connection.post("/?#{@config.except(:database).to_param}", sql)
315
- process_response(res)
315
+ res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
316
+ process_response(res, DEFAULT_RESPONSE_FORMAT)
316
317
  end
317
318
  end
318
319
 
@@ -365,15 +366,15 @@ module ActiveRecord
365
366
  end
366
367
 
367
368
  def cluster
368
- @full_config[:cluster_name]
369
+ @config[:cluster_name]
369
370
  end
370
371
 
371
372
  def replica
372
- @full_config[:replica_name]
373
+ @config[:replica_name]
373
374
  end
374
375
 
375
376
  def use_default_replicated_merge_tree_params?
376
- database_engine_atomic? && @full_config[:use_default_replicated_merge_tree_params]
377
+ database_engine_atomic? && @config[:use_default_replicated_merge_tree_params]
377
378
  end
378
379
 
379
380
  def use_replica?
@@ -381,11 +382,11 @@ module ActiveRecord
381
382
  end
382
383
 
383
384
  def replica_path(table)
384
- "/clickhouse/tables/#{cluster}/#{@config[:database]}.#{table}"
385
+ "/clickhouse/tables/#{cluster}/#{@connection_config[:database]}.#{table}"
385
386
  end
386
387
 
387
388
  def database_engine_atomic?
388
- current_database_engine = "select engine from system.databases where name = '#{@config[:database]}'"
389
+ current_database_engine = "select engine from system.databases where name = '#{@connection_config[:database]}'"
389
390
  res = select_one(current_database_engine)
390
391
  res['engine'] == 'Atomic' if res
391
392
  end
@@ -13,6 +13,13 @@ module Arel
13
13
  end
14
14
  end
15
15
 
16
+ # https://clickhouse.com/docs/en/sql-reference/statements/delete
17
+ # DELETE and UPDATE in ClickHouse working only without table name
18
+ def visit_Arel_Attributes_Attribute(o, collector)
19
+ collector << quote_table_name(o.relation.table_alias || o.relation.name) << '.' unless collector.value.start_with?('DELETE FROM ') || collector.value.include?(' UPDATE ')
20
+ collector << quote_column_name(o.name)
21
+ end
22
+
16
23
  def visit_Arel_Nodes_SelectOptions(o, collector)
17
24
  maybe_visit o.settings, super
18
25
  end
@@ -53,6 +60,16 @@ module Arel
53
60
  collector
54
61
  end
55
62
 
63
+ def visit_Arel_Nodes_Matches(o, collector)
64
+ op = o.case_sensitive ? " LIKE " : " ILIKE "
65
+ infix_value o, collector, op
66
+ end
67
+
68
+ def visit_Arel_Nodes_DoesNotMatch(o, collector)
69
+ op = o.case_sensitive ? " NOT LIKE " : " NOT ILIKE "
70
+ infix_value o, collector, op
71
+ end
72
+
56
73
  def sanitize_as_setting_value(value)
57
74
  if value == :default
58
75
  'DEFAULT'
@@ -1,3 +1,3 @@
1
1
  module ClickhouseActiverecord
2
- VERSION = '1.0.3'
2
+ VERSION = '1.0.5'
3
3
  end
@@ -24,9 +24,9 @@ end
24
24
 
25
25
  module ClickhouseActiverecord
26
26
  def self.load
27
- ActiveRecord::InternalMetadata.singleton_class.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods)
27
+ ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods)
28
28
  ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation)
29
- ActiveRecord::SchemaMigration.singleton_class.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods)
29
+ ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods)
30
30
 
31
31
  Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore)
32
32
  Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement)
@@ -3,17 +3,6 @@ module CoreExtensions
3
3
  module InternalMetadata
4
4
  module ClassMethods
5
5
 
6
- def []=(key, value)
7
- row = final.find_by(key: key)
8
- if row.nil? || row.value != value
9
- create!(key: key, value: value)
10
- end
11
- end
12
-
13
- def [](key)
14
- final.where(key: key).pluck(:value).first
15
- end
16
-
17
6
  def create_table
18
7
  return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
19
8
  return if table_exists? || !enabled?
@@ -21,10 +10,10 @@ module CoreExtensions
21
10
  key_options = connection.internal_string_options_for_primary_key
22
11
  table_options = {
23
12
  id: false,
24
- options: connection.adapter_name.downcase == 'clickhouse' ? 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key' : '',
13
+ options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key',
25
14
  if_not_exists: true
26
15
  }
27
- full_config = connection.instance_variable_get(:@full_config) || {}
16
+ full_config = connection.instance_variable_get(:@config) || {}
28
17
 
29
18
  if full_config[:distributed_service_tables]
30
19
  table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
@@ -40,6 +29,27 @@ module CoreExtensions
40
29
  t.timestamps
41
30
  end
42
31
  end
32
+
33
+ private
34
+
35
+ def update_entry(key, new_value)
36
+ return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
37
+
38
+ create_entry(key, new_value)
39
+ end
40
+
41
+ def select_entry(key)
42
+ return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
43
+
44
+ sm = ::Arel::SelectManager.new(arel_table)
45
+ sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/
46
+ sm.project(::Arel.star)
47
+ sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key)))
48
+ sm.order(arel_table[primary_key].asc)
49
+ sm.limit = 1
50
+
51
+ connection.select_one(sm, "#{self.class} Load")
52
+ end
43
53
  end
44
54
  end
45
55
  end
@@ -12,7 +12,7 @@ module CoreExtensions
12
12
  table_options = {
13
13
  id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true
14
14
  }
15
- full_config = connection.instance_variable_get(:@full_config) || {}
15
+ full_config = connection.instance_variable_get(:@config) || {}
16
16
 
17
17
  if full_config[:distributed_service_tables]
18
18
  table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)')
@@ -32,7 +32,7 @@ module CoreExtensions
32
32
  def delete_version(version)
33
33
  return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
34
34
 
35
- im = Arel::InsertManager.new(arel_table)
35
+ im = ::Arel::InsertManager.new(arel_table)
36
36
  im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0)
37
37
  connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version)
38
38
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: clickhouse-activerecord
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sergey Odintsov
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-01-22 00:00:00.000000000 Z
11
+ date: 2024-03-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -87,6 +87,14 @@ executables: []
87
87
  extensions: []
88
88
  extra_rdoc_files: []
89
89
  files:
90
+ - ".docker/clickhouse/cluster/server1_config.xml"
91
+ - ".docker/clickhouse/cluster/server2_config.xml"
92
+ - ".docker/clickhouse/single/config.xml"
93
+ - ".docker/clickhouse/users.xml"
94
+ - ".docker/docker-compose.cluster.yml"
95
+ - ".docker/docker-compose.yml"
96
+ - ".docker/nginx/local.conf"
97
+ - ".github/workflows/testing.yml"
90
98
  - ".gitignore"
91
99
  - ".rspec"
92
100
  - CHANGELOG.md