clickhouse-activerecord 1.0.3 → 1.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.docker/clickhouse/cluster/server1_config.xml +117 -0
- data/.docker/clickhouse/cluster/server2_config.xml +117 -0
- data/.docker/clickhouse/single/config.xml +54 -0
- data/.docker/clickhouse/users.xml +34 -0
- data/.docker/docker-compose.cluster.yml +41 -0
- data/.docker/docker-compose.yml +14 -0
- data/.docker/nginx/local.conf +12 -0
- data/.github/workflows/testing.yml +77 -0
- data/CHANGELOG.md +28 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +76 -17
- data/lib/active_record/connection_adapters/clickhouse_adapter.rb +20 -19
- data/lib/arel/visitors/clickhouse.rb +17 -0
- data/lib/clickhouse-activerecord/version.rb +1 -1
- data/lib/clickhouse-activerecord.rb +2 -2
- data/lib/core_extensions/active_record/internal_metadata.rb +23 -13
- data/lib/core_extensions/active_record/schema_migration.rb +2 -2
- metadata +10 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2217c1539b22307398fbc9c01bb8b521300ca23a7d03c2d32a56a9abdd3c5546
|
4
|
+
data.tar.gz: 8aa60f1a78db38df4120b0b405ddaf9466da70e68bf382f58945b29d5e130b7b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1e48e302475eff9ca738b35fc56b58a7a73f7d6b6cf5ae6632a7ee95fbc9432734ba18c53f7756d833ebd3748368fd73f7c29e04a0972f3141d00b5acd262cd
|
7
|
+
data.tar.gz: acedbb7298cd044769a5a1deaaae44672bdbf1affca5d2218e8b7209731df505434ed4f69bb23881e0d4f231e9e6699ac621ab0b4ac19f86475dd6c6515b3e18
|
@@ -0,0 +1,117 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<clickhouse>
|
3
|
+
|
4
|
+
<http_port>8123</http_port>
|
5
|
+
<interserver_http_port>9009</interserver_http_port>
|
6
|
+
<interserver_http_host>clickhouse1</interserver_http_host>
|
7
|
+
|
8
|
+
<users_config>users.xml</users_config>
|
9
|
+
<default_profile>default</default_profile>
|
10
|
+
<default_database>default</default_database>
|
11
|
+
|
12
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
13
|
+
|
14
|
+
<path>/var/lib/clickhouse/</path>
|
15
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
16
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
17
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
18
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
19
|
+
|
20
|
+
<logger>
|
21
|
+
<level>debug</level>
|
22
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
23
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
24
|
+
<size>1000M</size>
|
25
|
+
<count>10</count>
|
26
|
+
<console>1</console>
|
27
|
+
</logger>
|
28
|
+
|
29
|
+
<remote_servers>
|
30
|
+
<test_cluster>
|
31
|
+
<shard>
|
32
|
+
<replica>
|
33
|
+
<host>clickhouse1</host>
|
34
|
+
<port>9000</port>
|
35
|
+
</replica>
|
36
|
+
<replica>
|
37
|
+
<host>clickhouse2</host>
|
38
|
+
<port>9000</port>
|
39
|
+
</replica>
|
40
|
+
</shard>
|
41
|
+
</test_cluster>
|
42
|
+
</remote_servers>
|
43
|
+
|
44
|
+
<keeper_server>
|
45
|
+
<tcp_port>9181</tcp_port>
|
46
|
+
<server_id>1</server_id>
|
47
|
+
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
48
|
+
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
49
|
+
|
50
|
+
<coordination_settings>
|
51
|
+
<operation_timeout_ms>10000</operation_timeout_ms>
|
52
|
+
<session_timeout_ms>30000</session_timeout_ms>
|
53
|
+
<raft_logs_level>trace</raft_logs_level>
|
54
|
+
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
|
55
|
+
</coordination_settings>
|
56
|
+
|
57
|
+
<raft_configuration>
|
58
|
+
<server>
|
59
|
+
<id>1</id>
|
60
|
+
<hostname>clickhouse1</hostname>
|
61
|
+
<port>9000</port>
|
62
|
+
</server>
|
63
|
+
<server>
|
64
|
+
<id>2</id>
|
65
|
+
<hostname>clickhouse2</hostname>
|
66
|
+
<port>9000</port>
|
67
|
+
</server>
|
68
|
+
</raft_configuration>
|
69
|
+
</keeper_server>
|
70
|
+
|
71
|
+
<zookeeper>
|
72
|
+
<node>
|
73
|
+
<host>clickhouse1</host>
|
74
|
+
<port>9181</port>
|
75
|
+
</node>
|
76
|
+
<node>
|
77
|
+
<host>clickhouse2</host>
|
78
|
+
<port>9181</port>
|
79
|
+
</node>
|
80
|
+
</zookeeper>
|
81
|
+
|
82
|
+
<macros>
|
83
|
+
<cluster>test_cluster</cluster>
|
84
|
+
<replica>clickhouse1</replica>
|
85
|
+
<shard>1</shard>
|
86
|
+
</macros>
|
87
|
+
|
88
|
+
<distributed_ddl>
|
89
|
+
<path>/clickhouse/test_cluster/task_queue/ddl</path>
|
90
|
+
</distributed_ddl>
|
91
|
+
|
92
|
+
<query_log>
|
93
|
+
<database>system</database>
|
94
|
+
<table>query_log</table>
|
95
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
96
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
97
|
+
</query_log>
|
98
|
+
|
99
|
+
<http_options_response>
|
100
|
+
<header>
|
101
|
+
<name>Access-Control-Allow-Origin</name>
|
102
|
+
<value>*</value>
|
103
|
+
</header>
|
104
|
+
<header>
|
105
|
+
<name>Access-Control-Allow-Headers</name>
|
106
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
107
|
+
</header>
|
108
|
+
<header>
|
109
|
+
<name>Access-Control-Allow-Methods</name>
|
110
|
+
<value>POST, GET, OPTIONS</value>
|
111
|
+
</header>
|
112
|
+
<header>
|
113
|
+
<name>Access-Control-Max-Age</name>
|
114
|
+
<value>86400</value>
|
115
|
+
</header>
|
116
|
+
</http_options_response>
|
117
|
+
</clickhouse>
|
@@ -0,0 +1,117 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<clickhouse>
|
3
|
+
|
4
|
+
<http_port>8123</http_port>
|
5
|
+
<interserver_http_port>9009</interserver_http_port>
|
6
|
+
<interserver_http_host>clickhouse2</interserver_http_host>
|
7
|
+
|
8
|
+
<users_config>users.xml</users_config>
|
9
|
+
<default_profile>default</default_profile>
|
10
|
+
<default_database>default</default_database>
|
11
|
+
|
12
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
13
|
+
|
14
|
+
<path>/var/lib/clickhouse/</path>
|
15
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
16
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
17
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
18
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
19
|
+
|
20
|
+
<logger>
|
21
|
+
<level>debug</level>
|
22
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
23
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
24
|
+
<size>1000M</size>
|
25
|
+
<count>10</count>
|
26
|
+
<console>1</console>
|
27
|
+
</logger>
|
28
|
+
|
29
|
+
<remote_servers>
|
30
|
+
<test_cluster>
|
31
|
+
<shard>
|
32
|
+
<replica>
|
33
|
+
<host>clickhouse1</host>
|
34
|
+
<port>9000</port>
|
35
|
+
</replica>
|
36
|
+
<replica>
|
37
|
+
<host>clickhouse2</host>
|
38
|
+
<port>9000</port>
|
39
|
+
</replica>
|
40
|
+
</shard>
|
41
|
+
</test_cluster>
|
42
|
+
</remote_servers>
|
43
|
+
|
44
|
+
<keeper_server>
|
45
|
+
<tcp_port>9181</tcp_port>
|
46
|
+
<server_id>2</server_id>
|
47
|
+
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
48
|
+
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
49
|
+
|
50
|
+
<coordination_settings>
|
51
|
+
<operation_timeout_ms>10000</operation_timeout_ms>
|
52
|
+
<session_timeout_ms>30000</session_timeout_ms>
|
53
|
+
<raft_logs_level>trace</raft_logs_level>
|
54
|
+
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
|
55
|
+
</coordination_settings>
|
56
|
+
|
57
|
+
<raft_configuration>
|
58
|
+
<server>
|
59
|
+
<id>1</id>
|
60
|
+
<hostname>clickhouse1</hostname>
|
61
|
+
<port>9000</port>
|
62
|
+
</server>
|
63
|
+
<server>
|
64
|
+
<id>2</id>
|
65
|
+
<hostname>clickhouse2</hostname>
|
66
|
+
<port>9000</port>
|
67
|
+
</server>
|
68
|
+
</raft_configuration>
|
69
|
+
</keeper_server>
|
70
|
+
|
71
|
+
<zookeeper>
|
72
|
+
<node>
|
73
|
+
<host>clickhouse1</host>
|
74
|
+
<port>9181</port>
|
75
|
+
</node>
|
76
|
+
<node>
|
77
|
+
<host>clickhouse2</host>
|
78
|
+
<port>9181</port>
|
79
|
+
</node>
|
80
|
+
</zookeeper>
|
81
|
+
|
82
|
+
<macros>
|
83
|
+
<cluster>test_cluster</cluster>
|
84
|
+
<replica>clickhouse2</replica>
|
85
|
+
<shard>1</shard>
|
86
|
+
</macros>
|
87
|
+
|
88
|
+
<distributed_ddl>
|
89
|
+
<path>/clickhouse/test_cluster/task_queue/ddl</path>
|
90
|
+
</distributed_ddl>
|
91
|
+
|
92
|
+
<query_log>
|
93
|
+
<database>system</database>
|
94
|
+
<table>query_log</table>
|
95
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
96
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
97
|
+
</query_log>
|
98
|
+
|
99
|
+
<http_options_response>
|
100
|
+
<header>
|
101
|
+
<name>Access-Control-Allow-Origin</name>
|
102
|
+
<value>*</value>
|
103
|
+
</header>
|
104
|
+
<header>
|
105
|
+
<name>Access-Control-Allow-Headers</name>
|
106
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
107
|
+
</header>
|
108
|
+
<header>
|
109
|
+
<name>Access-Control-Allow-Methods</name>
|
110
|
+
<value>POST, GET, OPTIONS</value>
|
111
|
+
</header>
|
112
|
+
<header>
|
113
|
+
<name>Access-Control-Max-Age</name>
|
114
|
+
<value>86400</value>
|
115
|
+
</header>
|
116
|
+
</http_options_response>
|
117
|
+
</clickhouse>
|
@@ -0,0 +1,54 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<clickhouse>
|
3
|
+
|
4
|
+
<http_port>8123</http_port>
|
5
|
+
<tcp_port>9000</tcp_port>
|
6
|
+
|
7
|
+
<users_config>users.xml</users_config>
|
8
|
+
<default_profile>default</default_profile>
|
9
|
+
<default_database>default</default_database>
|
10
|
+
|
11
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
12
|
+
|
13
|
+
<path>/var/lib/clickhouse/</path>
|
14
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
15
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
16
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
17
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
18
|
+
|
19
|
+
<logger>
|
20
|
+
<level>debug</level>
|
21
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
22
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
23
|
+
<size>1000M</size>
|
24
|
+
<count>10</count>
|
25
|
+
<console>1</console>
|
26
|
+
</logger>
|
27
|
+
|
28
|
+
<query_log>
|
29
|
+
<database>system</database>
|
30
|
+
<table>query_log</table>
|
31
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
32
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
33
|
+
</query_log>
|
34
|
+
|
35
|
+
<http_options_response>
|
36
|
+
<header>
|
37
|
+
<name>Access-Control-Allow-Origin</name>
|
38
|
+
<value>*</value>
|
39
|
+
</header>
|
40
|
+
<header>
|
41
|
+
<name>Access-Control-Allow-Headers</name>
|
42
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
43
|
+
</header>
|
44
|
+
<header>
|
45
|
+
<name>Access-Control-Allow-Methods</name>
|
46
|
+
<value>POST, GET, OPTIONS</value>
|
47
|
+
</header>
|
48
|
+
<header>
|
49
|
+
<name>Access-Control-Max-Age</name>
|
50
|
+
<value>86400</value>
|
51
|
+
</header>
|
52
|
+
</http_options_response>
|
53
|
+
|
54
|
+
</clickhouse>
|
@@ -0,0 +1,34 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<clickhouse>
|
3
|
+
|
4
|
+
<profiles>
|
5
|
+
<default>
|
6
|
+
<load_balancing>random</load_balancing>
|
7
|
+
</default>
|
8
|
+
</profiles>
|
9
|
+
|
10
|
+
<users>
|
11
|
+
<default>
|
12
|
+
<password></password>
|
13
|
+
<networks>
|
14
|
+
<ip>::/0</ip>
|
15
|
+
</networks>
|
16
|
+
<profile>default</profile>
|
17
|
+
<quota>default</quota>
|
18
|
+
<access_management>1</access_management>
|
19
|
+
</default>
|
20
|
+
</users>
|
21
|
+
|
22
|
+
<quotas>
|
23
|
+
<default>
|
24
|
+
<interval>
|
25
|
+
<duration>3600</duration>
|
26
|
+
<queries>0</queries>
|
27
|
+
<errors>0</errors>
|
28
|
+
<result_rows>0</result_rows>
|
29
|
+
<read_rows>0</read_rows>
|
30
|
+
<execution_time>0</execution_time>
|
31
|
+
</interval>
|
32
|
+
</default>
|
33
|
+
</quotas>
|
34
|
+
</clickhouse>
|
@@ -0,0 +1,41 @@
|
|
1
|
+
version: '3.5'
|
2
|
+
|
3
|
+
services:
|
4
|
+
clickhouse1:
|
5
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
6
|
+
ulimits:
|
7
|
+
nofile:
|
8
|
+
soft: 262144
|
9
|
+
hard: 262144
|
10
|
+
hostname: clickhouse1
|
11
|
+
container_name: clickhouse-activerecord-clickhouse-server-1
|
12
|
+
ports:
|
13
|
+
- '8124:8123'
|
14
|
+
- '9001:9000'
|
15
|
+
volumes:
|
16
|
+
- './clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml'
|
17
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
18
|
+
|
19
|
+
clickhouse2:
|
20
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
21
|
+
ulimits:
|
22
|
+
nofile:
|
23
|
+
soft: 262144
|
24
|
+
hard: 262144
|
25
|
+
hostname: clickhouse2
|
26
|
+
container_name: clickhouse-activerecord-clickhouse-server-2
|
27
|
+
ports:
|
28
|
+
- '8125:8123'
|
29
|
+
volumes:
|
30
|
+
- './clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml'
|
31
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
32
|
+
|
33
|
+
# Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests
|
34
|
+
nginx:
|
35
|
+
image: 'nginx:1.23.1-alpine'
|
36
|
+
hostname: nginx
|
37
|
+
ports:
|
38
|
+
- '28123:8123'
|
39
|
+
volumes:
|
40
|
+
- './nginx/local.conf:/etc/nginx/conf.d/local.conf'
|
41
|
+
container_name: clickhouse-activerecord-nginx
|
@@ -0,0 +1,14 @@
|
|
1
|
+
version: '3.8'
|
2
|
+
services:
|
3
|
+
clickhouse:
|
4
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
5
|
+
container_name: 'clickhouse-activerecord-clickhouse-server'
|
6
|
+
ports:
|
7
|
+
- '18123:8123'
|
8
|
+
ulimits:
|
9
|
+
nofile:
|
10
|
+
soft: 262144
|
11
|
+
hard: 262144
|
12
|
+
volumes:
|
13
|
+
- './clickhouse/single/config.xml:/etc/clickhouse-server/config.xml'
|
14
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
@@ -0,0 +1,77 @@
|
|
1
|
+
name: Testing
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches: [ "master" ]
|
6
|
+
pull_request:
|
7
|
+
branches: [ "master" ]
|
8
|
+
|
9
|
+
jobs:
|
10
|
+
tests_single:
|
11
|
+
name: Testing single server
|
12
|
+
runs-on: ubuntu-latest
|
13
|
+
|
14
|
+
env:
|
15
|
+
CLICKHOUSE_PORT: 18123
|
16
|
+
CLICKHOUSE_DATABASE: default
|
17
|
+
|
18
|
+
strategy:
|
19
|
+
fail-fast: true
|
20
|
+
max-parallel: 1
|
21
|
+
matrix:
|
22
|
+
ruby-version: [ '2.7', '3.0', '3.2' ]
|
23
|
+
clickhouse: [ '22.1' ]
|
24
|
+
|
25
|
+
steps:
|
26
|
+
- uses: actions/checkout@v4
|
27
|
+
|
28
|
+
- name: Start ClickHouse ${{ matrix.clickhouse }}
|
29
|
+
uses: isbang/compose-action@v1.5.1
|
30
|
+
env:
|
31
|
+
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
|
32
|
+
with:
|
33
|
+
compose-file: '.docker/docker-compose.yml'
|
34
|
+
down-flags: '--volumes'
|
35
|
+
|
36
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
37
|
+
uses: ruby/setup-ruby@v1
|
38
|
+
with:
|
39
|
+
ruby-version: ${{ matrix.ruby-version }}
|
40
|
+
bundler-cache: true
|
41
|
+
|
42
|
+
- run: bundle exec rspec spec/single
|
43
|
+
|
44
|
+
tests_cluster:
|
45
|
+
name: Testing cluster server
|
46
|
+
runs-on: ubuntu-latest
|
47
|
+
|
48
|
+
env:
|
49
|
+
CLICKHOUSE_PORT: 28123
|
50
|
+
CLICKHOUSE_DATABASE: default
|
51
|
+
CLICKHOUSE_CLUSTER: test_cluster
|
52
|
+
|
53
|
+
strategy:
|
54
|
+
fail-fast: true
|
55
|
+
max-parallel: 1
|
56
|
+
matrix:
|
57
|
+
ruby-version: [ '2.7', '3.0', '3.2' ]
|
58
|
+
clickhouse: [ '22.1' ]
|
59
|
+
|
60
|
+
steps:
|
61
|
+
- uses: actions/checkout@v4
|
62
|
+
|
63
|
+
- name: Start ClickHouse Cluster ${{ matrix.clickhouse }}
|
64
|
+
uses: isbang/compose-action@v1.5.1
|
65
|
+
env:
|
66
|
+
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
|
67
|
+
with:
|
68
|
+
compose-file: '.docker/docker-compose.cluster.yml'
|
69
|
+
down-flags: '--volumes'
|
70
|
+
|
71
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
72
|
+
uses: ruby/setup-ruby@v1
|
73
|
+
with:
|
74
|
+
ruby-version: ${{ matrix.ruby-version }}
|
75
|
+
bundler-cache: true
|
76
|
+
|
77
|
+
- run: bundle exec rspec spec/cluster
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,31 @@
|
|
1
|
+
### Version 1.0.5 (Mar 14, 2024)
|
2
|
+
|
3
|
+
* GitHub workflows
|
4
|
+
* Fix injection internal and schema classes for rails 7
|
5
|
+
* Add support for binary string by [@PauloMiranda98](https://github.com/PauloMiranda98) in (#116)
|
6
|
+
|
7
|
+
### Version 1.0.4 (Feb 2, 2024)
|
8
|
+
|
9
|
+
* Use ILIKE for `model.arel_table[:column]#matches` by [@stympy](https://github.com/stympy) in (#115)
|
10
|
+
* Fixed `insert_all` for array column (#71)
|
11
|
+
* Register Bool and UUID in type map by [@lukinski](https://github.com/lukinski) in (#110)
|
12
|
+
* Refactoring `final` method
|
13
|
+
* Support update & delete for clickhouse from version 23.3 and newer (#93)
|
14
|
+
|
15
|
+
### Version 1.0.0 (Nov 29, 2023)
|
16
|
+
|
17
|
+
* Full support Rails 7.1+
|
18
|
+
* Full support primary or multiple databases
|
19
|
+
|
20
|
+
### Version 0.6.0 (Oct 19, 2023)
|
21
|
+
|
22
|
+
* Added `Bool` column type instead `Uint8` (#78). Supports ClickHouse 22+ database only
|
23
|
+
* Added `final` method (#81) (The `ar_internal_metadata` table needs to be deleted after a gem update)
|
24
|
+
* Added `settings` method (#82)
|
25
|
+
* Fixed convert aggregation type (#92)
|
26
|
+
* Fixed raise error not database exist (#91)
|
27
|
+
* Fixed internal metadata update (#84)
|
28
|
+
|
1
29
|
### Version 0.5.10 (Jun 22, 2022)
|
2
30
|
|
3
31
|
* Fixes to create_table method (#70)
|
@@ -6,6 +6,8 @@ module ActiveRecord
|
|
6
6
|
module ConnectionAdapters
|
7
7
|
module Clickhouse
|
8
8
|
module SchemaStatements
|
9
|
+
DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
|
10
|
+
|
9
11
|
def execute(sql, name = nil, settings: {})
|
10
12
|
do_execute(sql, name, settings: settings)
|
11
13
|
end
|
@@ -33,13 +35,20 @@ module ActiveRecord
|
|
33
35
|
# @link https://clickhouse.com/docs/en/sql-reference/statements/alter/update
|
34
36
|
def exec_update(_sql, _name = nil, _binds = [])
|
35
37
|
do_execute(_sql, _name, format: nil)
|
36
|
-
|
38
|
+
0
|
37
39
|
end
|
38
40
|
|
39
41
|
# @link https://clickhouse.com/docs/en/sql-reference/statements/delete
|
40
42
|
def exec_delete(_sql, _name = nil, _binds = [])
|
41
|
-
|
42
|
-
|
43
|
+
log(_sql, "#{adapter_name} #{_name}") do
|
44
|
+
res = request(_sql)
|
45
|
+
begin
|
46
|
+
data = JSON.parse(res.header['x-clickhouse-summary'])
|
47
|
+
data['result_rows'].to_i
|
48
|
+
rescue JSONError
|
49
|
+
0
|
50
|
+
end
|
51
|
+
end
|
43
52
|
end
|
44
53
|
|
45
54
|
def tables(name = nil)
|
@@ -64,19 +73,15 @@ module ActiveRecord
|
|
64
73
|
|
65
74
|
def do_system_execute(sql, name = nil)
|
66
75
|
log_with_debug(sql, "#{adapter_name} #{name}") do
|
67
|
-
res =
|
68
|
-
|
69
|
-
process_response(res)
|
76
|
+
res = request(sql, DEFAULT_RESPONSE_FORMAT)
|
77
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
70
78
|
end
|
71
79
|
end
|
72
80
|
|
73
|
-
def do_execute(sql, name = nil, format:
|
81
|
+
def do_execute(sql, name = nil, format: DEFAULT_RESPONSE_FORMAT, settings: {})
|
74
82
|
log(sql, "#{adapter_name} #{name}") do
|
75
|
-
|
76
|
-
|
77
|
-
res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
|
78
|
-
|
79
|
-
process_response(res)
|
83
|
+
res = request(sql, format, settings)
|
84
|
+
process_response(res, format)
|
80
85
|
end
|
81
86
|
end
|
82
87
|
|
@@ -96,23 +101,44 @@ module ActiveRecord
|
|
96
101
|
if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
|
97
102
|
raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
|
98
103
|
end
|
99
|
-
do_execute(insert_versions_sql(inserting), nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
|
104
|
+
do_execute(insert_versions_sql(inserting), nil, format: nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
# Fix insert_all method
|
109
|
+
# https://github.com/PNixx/clickhouse-activerecord/issues/71#issuecomment-1923244983
|
110
|
+
def with_yaml_fallback(value) # :nodoc:
|
111
|
+
if value.is_a?(Array)
|
112
|
+
value
|
113
|
+
else
|
114
|
+
super
|
100
115
|
end
|
101
116
|
end
|
102
117
|
|
103
118
|
private
|
104
119
|
|
120
|
+
# Make HTTP request to ClickHouse server
|
121
|
+
# @param [String] sql
|
122
|
+
# @param [String, nil] format
|
123
|
+
# @param [Hash] settings
|
124
|
+
# @return [Net::HTTPResponse]
|
125
|
+
def request(sql, format = nil, settings = {})
|
126
|
+
formatted_sql = apply_format(sql, format)
|
127
|
+
request_params = @connection_config || {}
|
128
|
+
@connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
|
129
|
+
end
|
130
|
+
|
105
131
|
def apply_format(sql, format)
|
106
132
|
format ? "#{sql} FORMAT #{format}" : sql
|
107
133
|
end
|
108
134
|
|
109
|
-
def process_response(res)
|
135
|
+
def process_response(res, format)
|
110
136
|
case res.code.to_i
|
111
137
|
when 200
|
112
138
|
if res.body.to_s.include?("DB::Exception")
|
113
139
|
raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}"
|
114
140
|
else
|
115
|
-
res.body
|
141
|
+
format_body_response(res.body, format)
|
116
142
|
end
|
117
143
|
else
|
118
144
|
case res.body
|
@@ -158,8 +184,7 @@ module ActiveRecord
|
|
158
184
|
|
159
185
|
return data unless data.empty?
|
160
186
|
|
161
|
-
raise ActiveRecord::StatementInvalid,
|
162
|
-
"Could not find table '#{table_name}'"
|
187
|
+
raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'"
|
163
188
|
end
|
164
189
|
alias column_definitions table_structure
|
165
190
|
|
@@ -193,6 +218,40 @@ module ActiveRecord
|
|
193
218
|
def has_default_function?(default_value, default) # :nodoc:
|
194
219
|
!default_value && (%r{\w+\(.*\)} === default)
|
195
220
|
end
|
221
|
+
|
222
|
+
def format_body_response(body, format)
|
223
|
+
return body if body.blank?
|
224
|
+
|
225
|
+
case format
|
226
|
+
when 'JSONCompact'
|
227
|
+
format_from_json_compact(body)
|
228
|
+
when 'JSONCompactEachRowWithNamesAndTypes'
|
229
|
+
format_from_json_compact_each_row_with_names_and_types(body)
|
230
|
+
else
|
231
|
+
body
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
def format_from_json_compact(body)
|
236
|
+
JSON.parse(body)
|
237
|
+
end
|
238
|
+
|
239
|
+
def format_from_json_compact_each_row_with_names_and_types(body)
|
240
|
+
rows = body.split("\n").map { |row| JSON.parse(row) }
|
241
|
+
names, types, *data = rows
|
242
|
+
|
243
|
+
meta = names.zip(types).map do |name, type|
|
244
|
+
{
|
245
|
+
'name' => name,
|
246
|
+
'type' => type
|
247
|
+
}
|
248
|
+
end
|
249
|
+
|
250
|
+
{
|
251
|
+
'meta' => meta,
|
252
|
+
'data' => data
|
253
|
+
}
|
254
|
+
end
|
196
255
|
end
|
197
256
|
end
|
198
257
|
end
|
@@ -45,7 +45,7 @@ module ActiveRecord
|
|
45
45
|
raise ArgumentError, 'No database specified. Missing argument: database.'
|
46
46
|
end
|
47
47
|
|
48
|
-
ConnectionAdapters::ClickhouseAdapter.new(logger, connection,
|
48
|
+
ConnectionAdapters::ClickhouseAdapter.new(logger, connection, config)
|
49
49
|
end
|
50
50
|
end
|
51
51
|
end
|
@@ -73,10 +73,11 @@ module ActiveRecord
|
|
73
73
|
def is_view=(value)
|
74
74
|
@is_view = value
|
75
75
|
end
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
76
|
+
|
77
|
+
def _delete_record(constraints)
|
78
|
+
raise ActiveRecord::ActiveRecordError.new('Deleting a row is not possible without a primary key') unless self.primary_key
|
79
|
+
super
|
80
|
+
end
|
80
81
|
end
|
81
82
|
end
|
82
83
|
|
@@ -120,12 +121,12 @@ module ActiveRecord
|
|
120
121
|
include Clickhouse::SchemaStatements
|
121
122
|
|
122
123
|
# Initializes and connects a Clickhouse adapter.
|
123
|
-
def initialize(logger, connection_parameters, config
|
124
|
+
def initialize(logger, connection_parameters, config)
|
124
125
|
super(nil, logger)
|
125
126
|
@connection_parameters = connection_parameters
|
127
|
+
@connection_config = { user: config[:username], password: config[:password], database: config[:database] }.compact
|
128
|
+
@debug = config[:debug] || false
|
126
129
|
@config = config
|
127
|
-
@debug = full_config[:debug] || false
|
128
|
-
@full_config = full_config
|
129
130
|
|
130
131
|
@prepared_statements = false
|
131
132
|
|
@@ -133,7 +134,7 @@ module ActiveRecord
|
|
133
134
|
end
|
134
135
|
|
135
136
|
def migrations_paths
|
136
|
-
@
|
137
|
+
@config[:migrations_paths] || 'db/migrate_clickhouse'
|
137
138
|
end
|
138
139
|
|
139
140
|
def arel_visitor # :nodoc:
|
@@ -266,8 +267,8 @@ module ActiveRecord
|
|
266
267
|
def create_database(name)
|
267
268
|
sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
|
268
269
|
log_with_debug(sql, adapter_name) do
|
269
|
-
res = @connection.post("/?#{@
|
270
|
-
process_response(res)
|
270
|
+
res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
|
271
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
271
272
|
end
|
272
273
|
end
|
273
274
|
|
@@ -302,7 +303,7 @@ module ActiveRecord
|
|
302
303
|
raise 'Set a cluster' unless cluster
|
303
304
|
|
304
305
|
distributed_options =
|
305
|
-
"Distributed(#{cluster}, #{@
|
306
|
+
"Distributed(#{cluster}, #{@connection_config[:database]}, #{table_name}, #{sharding_key})"
|
306
307
|
create_table(distributed_table_name, **options.merge(options: distributed_options), &block)
|
307
308
|
end
|
308
309
|
end
|
@@ -311,8 +312,8 @@ module ActiveRecord
|
|
311
312
|
def drop_database(name) #:nodoc:
|
312
313
|
sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
|
313
314
|
log_with_debug(sql, adapter_name) do
|
314
|
-
res = @connection.post("/?#{@
|
315
|
-
process_response(res)
|
315
|
+
res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
|
316
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
316
317
|
end
|
317
318
|
end
|
318
319
|
|
@@ -365,15 +366,15 @@ module ActiveRecord
|
|
365
366
|
end
|
366
367
|
|
367
368
|
def cluster
|
368
|
-
@
|
369
|
+
@config[:cluster_name]
|
369
370
|
end
|
370
371
|
|
371
372
|
def replica
|
372
|
-
@
|
373
|
+
@config[:replica_name]
|
373
374
|
end
|
374
375
|
|
375
376
|
def use_default_replicated_merge_tree_params?
|
376
|
-
database_engine_atomic? && @
|
377
|
+
database_engine_atomic? && @config[:use_default_replicated_merge_tree_params]
|
377
378
|
end
|
378
379
|
|
379
380
|
def use_replica?
|
@@ -381,11 +382,11 @@ module ActiveRecord
|
|
381
382
|
end
|
382
383
|
|
383
384
|
def replica_path(table)
|
384
|
-
"/clickhouse/tables/#{cluster}/#{@
|
385
|
+
"/clickhouse/tables/#{cluster}/#{@connection_config[:database]}.#{table}"
|
385
386
|
end
|
386
387
|
|
387
388
|
def database_engine_atomic?
|
388
|
-
current_database_engine = "select engine from system.databases where name = '#{@
|
389
|
+
current_database_engine = "select engine from system.databases where name = '#{@connection_config[:database]}'"
|
389
390
|
res = select_one(current_database_engine)
|
390
391
|
res['engine'] == 'Atomic' if res
|
391
392
|
end
|
@@ -13,6 +13,13 @@ module Arel
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
|
16
|
+
# https://clickhouse.com/docs/en/sql-reference/statements/delete
|
17
|
+
# DELETE and UPDATE in ClickHouse working only without table name
|
18
|
+
def visit_Arel_Attributes_Attribute(o, collector)
|
19
|
+
collector << quote_table_name(o.relation.table_alias || o.relation.name) << '.' unless collector.value.start_with?('DELETE FROM ') || collector.value.include?(' UPDATE ')
|
20
|
+
collector << quote_column_name(o.name)
|
21
|
+
end
|
22
|
+
|
16
23
|
def visit_Arel_Nodes_SelectOptions(o, collector)
|
17
24
|
maybe_visit o.settings, super
|
18
25
|
end
|
@@ -53,6 +60,16 @@ module Arel
|
|
53
60
|
collector
|
54
61
|
end
|
55
62
|
|
63
|
+
def visit_Arel_Nodes_Matches(o, collector)
|
64
|
+
op = o.case_sensitive ? " LIKE " : " ILIKE "
|
65
|
+
infix_value o, collector, op
|
66
|
+
end
|
67
|
+
|
68
|
+
def visit_Arel_Nodes_DoesNotMatch(o, collector)
|
69
|
+
op = o.case_sensitive ? " NOT LIKE " : " NOT ILIKE "
|
70
|
+
infix_value o, collector, op
|
71
|
+
end
|
72
|
+
|
56
73
|
def sanitize_as_setting_value(value)
|
57
74
|
if value == :default
|
58
75
|
'DEFAULT'
|
@@ -24,9 +24,9 @@ end
|
|
24
24
|
|
25
25
|
module ClickhouseActiverecord
|
26
26
|
def self.load
|
27
|
-
ActiveRecord::InternalMetadata.
|
27
|
+
ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods)
|
28
28
|
ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation)
|
29
|
-
ActiveRecord::SchemaMigration.
|
29
|
+
ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods)
|
30
30
|
|
31
31
|
Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore)
|
32
32
|
Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement)
|
@@ -3,17 +3,6 @@ module CoreExtensions
|
|
3
3
|
module InternalMetadata
|
4
4
|
module ClassMethods
|
5
5
|
|
6
|
-
def []=(key, value)
|
7
|
-
row = final.find_by(key: key)
|
8
|
-
if row.nil? || row.value != value
|
9
|
-
create!(key: key, value: value)
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
def [](key)
|
14
|
-
final.where(key: key).pluck(:value).first
|
15
|
-
end
|
16
|
-
|
17
6
|
def create_table
|
18
7
|
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
19
8
|
return if table_exists? || !enabled?
|
@@ -21,10 +10,10 @@ module CoreExtensions
|
|
21
10
|
key_options = connection.internal_string_options_for_primary_key
|
22
11
|
table_options = {
|
23
12
|
id: false,
|
24
|
-
options:
|
13
|
+
options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key',
|
25
14
|
if_not_exists: true
|
26
15
|
}
|
27
|
-
full_config = connection.instance_variable_get(:@
|
16
|
+
full_config = connection.instance_variable_get(:@config) || {}
|
28
17
|
|
29
18
|
if full_config[:distributed_service_tables]
|
30
19
|
table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
|
@@ -40,6 +29,27 @@ module CoreExtensions
|
|
40
29
|
t.timestamps
|
41
30
|
end
|
42
31
|
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def update_entry(key, new_value)
|
36
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
37
|
+
|
38
|
+
create_entry(key, new_value)
|
39
|
+
end
|
40
|
+
|
41
|
+
def select_entry(key)
|
42
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
43
|
+
|
44
|
+
sm = ::Arel::SelectManager.new(arel_table)
|
45
|
+
sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/
|
46
|
+
sm.project(::Arel.star)
|
47
|
+
sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key)))
|
48
|
+
sm.order(arel_table[primary_key].asc)
|
49
|
+
sm.limit = 1
|
50
|
+
|
51
|
+
connection.select_one(sm, "#{self.class} Load")
|
52
|
+
end
|
43
53
|
end
|
44
54
|
end
|
45
55
|
end
|
@@ -12,7 +12,7 @@ module CoreExtensions
|
|
12
12
|
table_options = {
|
13
13
|
id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true
|
14
14
|
}
|
15
|
-
full_config = connection.instance_variable_get(:@
|
15
|
+
full_config = connection.instance_variable_get(:@config) || {}
|
16
16
|
|
17
17
|
if full_config[:distributed_service_tables]
|
18
18
|
table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)')
|
@@ -32,7 +32,7 @@ module CoreExtensions
|
|
32
32
|
def delete_version(version)
|
33
33
|
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
34
34
|
|
35
|
-
im = Arel::InsertManager.new(arel_table)
|
35
|
+
im = ::Arel::InsertManager.new(arel_table)
|
36
36
|
im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0)
|
37
37
|
connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version)
|
38
38
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: clickhouse-activerecord
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sergey Odintsov
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-03-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -87,6 +87,14 @@ executables: []
|
|
87
87
|
extensions: []
|
88
88
|
extra_rdoc_files: []
|
89
89
|
files:
|
90
|
+
- ".docker/clickhouse/cluster/server1_config.xml"
|
91
|
+
- ".docker/clickhouse/cluster/server2_config.xml"
|
92
|
+
- ".docker/clickhouse/single/config.xml"
|
93
|
+
- ".docker/clickhouse/users.xml"
|
94
|
+
- ".docker/docker-compose.cluster.yml"
|
95
|
+
- ".docker/docker-compose.yml"
|
96
|
+
- ".docker/nginx/local.conf"
|
97
|
+
- ".github/workflows/testing.yml"
|
90
98
|
- ".gitignore"
|
91
99
|
- ".rspec"
|
92
100
|
- CHANGELOG.md
|