clickhouse-activerecord 1.0.4 → 1.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.docker/clickhouse/cluster/server1_config.xml +117 -0
- data/.docker/clickhouse/cluster/server2_config.xml +117 -0
- data/.docker/clickhouse/single/config.xml +54 -0
- data/.docker/clickhouse/users.xml +34 -0
- data/.docker/docker-compose.cluster.yml +41 -0
- data/.docker/docker-compose.yml +14 -0
- data/.docker/nginx/local.conf +12 -0
- data/.github/workflows/testing.yml +77 -0
- data/CHANGELOG.md +28 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +66 -17
- data/lib/active_record/connection_adapters/clickhouse_adapter.rb +7 -6
- data/lib/arel/visitors/clickhouse.rb +7 -0
- data/lib/clickhouse-activerecord/version.rb +1 -1
- data/lib/clickhouse-activerecord.rb +2 -2
- data/lib/core_extensions/active_record/internal_metadata.rb +22 -12
- data/lib/core_extensions/active_record/schema_migration.rb +1 -1
- metadata +10 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2217c1539b22307398fbc9c01bb8b521300ca23a7d03c2d32a56a9abdd3c5546
|
|
4
|
+
data.tar.gz: 8aa60f1a78db38df4120b0b405ddaf9466da70e68bf382f58945b29d5e130b7b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d1e48e302475eff9ca738b35fc56b58a7a73f7d6b6cf5ae6632a7ee95fbc9432734ba18c53f7756d833ebd3748368fd73f7c29e04a0972f3141d00b5acd262cd
|
|
7
|
+
data.tar.gz: acedbb7298cd044769a5a1deaaae44672bdbf1affca5d2218e8b7209731df505434ed4f69bb23881e0d4f231e9e6699ac621ab0b4ac19f86475dd6c6515b3e18
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<clickhouse>
|
|
3
|
+
|
|
4
|
+
<http_port>8123</http_port>
|
|
5
|
+
<interserver_http_port>9009</interserver_http_port>
|
|
6
|
+
<interserver_http_host>clickhouse1</interserver_http_host>
|
|
7
|
+
|
|
8
|
+
<users_config>users.xml</users_config>
|
|
9
|
+
<default_profile>default</default_profile>
|
|
10
|
+
<default_database>default</default_database>
|
|
11
|
+
|
|
12
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
|
13
|
+
|
|
14
|
+
<path>/var/lib/clickhouse/</path>
|
|
15
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
16
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
17
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
|
18
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
|
19
|
+
|
|
20
|
+
<logger>
|
|
21
|
+
<level>debug</level>
|
|
22
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
|
23
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
|
24
|
+
<size>1000M</size>
|
|
25
|
+
<count>10</count>
|
|
26
|
+
<console>1</console>
|
|
27
|
+
</logger>
|
|
28
|
+
|
|
29
|
+
<remote_servers>
|
|
30
|
+
<test_cluster>
|
|
31
|
+
<shard>
|
|
32
|
+
<replica>
|
|
33
|
+
<host>clickhouse1</host>
|
|
34
|
+
<port>9000</port>
|
|
35
|
+
</replica>
|
|
36
|
+
<replica>
|
|
37
|
+
<host>clickhouse2</host>
|
|
38
|
+
<port>9000</port>
|
|
39
|
+
</replica>
|
|
40
|
+
</shard>
|
|
41
|
+
</test_cluster>
|
|
42
|
+
</remote_servers>
|
|
43
|
+
|
|
44
|
+
<keeper_server>
|
|
45
|
+
<tcp_port>9181</tcp_port>
|
|
46
|
+
<server_id>1</server_id>
|
|
47
|
+
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
|
48
|
+
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
|
49
|
+
|
|
50
|
+
<coordination_settings>
|
|
51
|
+
<operation_timeout_ms>10000</operation_timeout_ms>
|
|
52
|
+
<session_timeout_ms>30000</session_timeout_ms>
|
|
53
|
+
<raft_logs_level>trace</raft_logs_level>
|
|
54
|
+
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
|
|
55
|
+
</coordination_settings>
|
|
56
|
+
|
|
57
|
+
<raft_configuration>
|
|
58
|
+
<server>
|
|
59
|
+
<id>1</id>
|
|
60
|
+
<hostname>clickhouse1</hostname>
|
|
61
|
+
<port>9000</port>
|
|
62
|
+
</server>
|
|
63
|
+
<server>
|
|
64
|
+
<id>2</id>
|
|
65
|
+
<hostname>clickhouse2</hostname>
|
|
66
|
+
<port>9000</port>
|
|
67
|
+
</server>
|
|
68
|
+
</raft_configuration>
|
|
69
|
+
</keeper_server>
|
|
70
|
+
|
|
71
|
+
<zookeeper>
|
|
72
|
+
<node>
|
|
73
|
+
<host>clickhouse1</host>
|
|
74
|
+
<port>9181</port>
|
|
75
|
+
</node>
|
|
76
|
+
<node>
|
|
77
|
+
<host>clickhouse2</host>
|
|
78
|
+
<port>9181</port>
|
|
79
|
+
</node>
|
|
80
|
+
</zookeeper>
|
|
81
|
+
|
|
82
|
+
<macros>
|
|
83
|
+
<cluster>test_cluster</cluster>
|
|
84
|
+
<replica>clickhouse1</replica>
|
|
85
|
+
<shard>1</shard>
|
|
86
|
+
</macros>
|
|
87
|
+
|
|
88
|
+
<distributed_ddl>
|
|
89
|
+
<path>/clickhouse/test_cluster/task_queue/ddl</path>
|
|
90
|
+
</distributed_ddl>
|
|
91
|
+
|
|
92
|
+
<query_log>
|
|
93
|
+
<database>system</database>
|
|
94
|
+
<table>query_log</table>
|
|
95
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
|
96
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
|
97
|
+
</query_log>
|
|
98
|
+
|
|
99
|
+
<http_options_response>
|
|
100
|
+
<header>
|
|
101
|
+
<name>Access-Control-Allow-Origin</name>
|
|
102
|
+
<value>*</value>
|
|
103
|
+
</header>
|
|
104
|
+
<header>
|
|
105
|
+
<name>Access-Control-Allow-Headers</name>
|
|
106
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
|
107
|
+
</header>
|
|
108
|
+
<header>
|
|
109
|
+
<name>Access-Control-Allow-Methods</name>
|
|
110
|
+
<value>POST, GET, OPTIONS</value>
|
|
111
|
+
</header>
|
|
112
|
+
<header>
|
|
113
|
+
<name>Access-Control-Max-Age</name>
|
|
114
|
+
<value>86400</value>
|
|
115
|
+
</header>
|
|
116
|
+
</http_options_response>
|
|
117
|
+
</clickhouse>
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<clickhouse>
|
|
3
|
+
|
|
4
|
+
<http_port>8123</http_port>
|
|
5
|
+
<interserver_http_port>9009</interserver_http_port>
|
|
6
|
+
<interserver_http_host>clickhouse2</interserver_http_host>
|
|
7
|
+
|
|
8
|
+
<users_config>users.xml</users_config>
|
|
9
|
+
<default_profile>default</default_profile>
|
|
10
|
+
<default_database>default</default_database>
|
|
11
|
+
|
|
12
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
|
13
|
+
|
|
14
|
+
<path>/var/lib/clickhouse/</path>
|
|
15
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
16
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
17
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
|
18
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
|
19
|
+
|
|
20
|
+
<logger>
|
|
21
|
+
<level>debug</level>
|
|
22
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
|
23
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
|
24
|
+
<size>1000M</size>
|
|
25
|
+
<count>10</count>
|
|
26
|
+
<console>1</console>
|
|
27
|
+
</logger>
|
|
28
|
+
|
|
29
|
+
<remote_servers>
|
|
30
|
+
<test_cluster>
|
|
31
|
+
<shard>
|
|
32
|
+
<replica>
|
|
33
|
+
<host>clickhouse1</host>
|
|
34
|
+
<port>9000</port>
|
|
35
|
+
</replica>
|
|
36
|
+
<replica>
|
|
37
|
+
<host>clickhouse2</host>
|
|
38
|
+
<port>9000</port>
|
|
39
|
+
</replica>
|
|
40
|
+
</shard>
|
|
41
|
+
</test_cluster>
|
|
42
|
+
</remote_servers>
|
|
43
|
+
|
|
44
|
+
<keeper_server>
|
|
45
|
+
<tcp_port>9181</tcp_port>
|
|
46
|
+
<server_id>2</server_id>
|
|
47
|
+
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
|
48
|
+
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
|
49
|
+
|
|
50
|
+
<coordination_settings>
|
|
51
|
+
<operation_timeout_ms>10000</operation_timeout_ms>
|
|
52
|
+
<session_timeout_ms>30000</session_timeout_ms>
|
|
53
|
+
<raft_logs_level>trace</raft_logs_level>
|
|
54
|
+
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
|
|
55
|
+
</coordination_settings>
|
|
56
|
+
|
|
57
|
+
<raft_configuration>
|
|
58
|
+
<server>
|
|
59
|
+
<id>1</id>
|
|
60
|
+
<hostname>clickhouse1</hostname>
|
|
61
|
+
<port>9000</port>
|
|
62
|
+
</server>
|
|
63
|
+
<server>
|
|
64
|
+
<id>2</id>
|
|
65
|
+
<hostname>clickhouse2</hostname>
|
|
66
|
+
<port>9000</port>
|
|
67
|
+
</server>
|
|
68
|
+
</raft_configuration>
|
|
69
|
+
</keeper_server>
|
|
70
|
+
|
|
71
|
+
<zookeeper>
|
|
72
|
+
<node>
|
|
73
|
+
<host>clickhouse1</host>
|
|
74
|
+
<port>9181</port>
|
|
75
|
+
</node>
|
|
76
|
+
<node>
|
|
77
|
+
<host>clickhouse2</host>
|
|
78
|
+
<port>9181</port>
|
|
79
|
+
</node>
|
|
80
|
+
</zookeeper>
|
|
81
|
+
|
|
82
|
+
<macros>
|
|
83
|
+
<cluster>test_cluster</cluster>
|
|
84
|
+
<replica>clickhouse2</replica>
|
|
85
|
+
<shard>1</shard>
|
|
86
|
+
</macros>
|
|
87
|
+
|
|
88
|
+
<distributed_ddl>
|
|
89
|
+
<path>/clickhouse/test_cluster/task_queue/ddl</path>
|
|
90
|
+
</distributed_ddl>
|
|
91
|
+
|
|
92
|
+
<query_log>
|
|
93
|
+
<database>system</database>
|
|
94
|
+
<table>query_log</table>
|
|
95
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
|
96
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
|
97
|
+
</query_log>
|
|
98
|
+
|
|
99
|
+
<http_options_response>
|
|
100
|
+
<header>
|
|
101
|
+
<name>Access-Control-Allow-Origin</name>
|
|
102
|
+
<value>*</value>
|
|
103
|
+
</header>
|
|
104
|
+
<header>
|
|
105
|
+
<name>Access-Control-Allow-Headers</name>
|
|
106
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
|
107
|
+
</header>
|
|
108
|
+
<header>
|
|
109
|
+
<name>Access-Control-Allow-Methods</name>
|
|
110
|
+
<value>POST, GET, OPTIONS</value>
|
|
111
|
+
</header>
|
|
112
|
+
<header>
|
|
113
|
+
<name>Access-Control-Max-Age</name>
|
|
114
|
+
<value>86400</value>
|
|
115
|
+
</header>
|
|
116
|
+
</http_options_response>
|
|
117
|
+
</clickhouse>
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<clickhouse>
|
|
3
|
+
|
|
4
|
+
<http_port>8123</http_port>
|
|
5
|
+
<tcp_port>9000</tcp_port>
|
|
6
|
+
|
|
7
|
+
<users_config>users.xml</users_config>
|
|
8
|
+
<default_profile>default</default_profile>
|
|
9
|
+
<default_database>default</default_database>
|
|
10
|
+
|
|
11
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
|
12
|
+
|
|
13
|
+
<path>/var/lib/clickhouse/</path>
|
|
14
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
15
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
16
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
|
17
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
|
18
|
+
|
|
19
|
+
<logger>
|
|
20
|
+
<level>debug</level>
|
|
21
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
|
22
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
|
23
|
+
<size>1000M</size>
|
|
24
|
+
<count>10</count>
|
|
25
|
+
<console>1</console>
|
|
26
|
+
</logger>
|
|
27
|
+
|
|
28
|
+
<query_log>
|
|
29
|
+
<database>system</database>
|
|
30
|
+
<table>query_log</table>
|
|
31
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
|
32
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
|
33
|
+
</query_log>
|
|
34
|
+
|
|
35
|
+
<http_options_response>
|
|
36
|
+
<header>
|
|
37
|
+
<name>Access-Control-Allow-Origin</name>
|
|
38
|
+
<value>*</value>
|
|
39
|
+
</header>
|
|
40
|
+
<header>
|
|
41
|
+
<name>Access-Control-Allow-Headers</name>
|
|
42
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
|
43
|
+
</header>
|
|
44
|
+
<header>
|
|
45
|
+
<name>Access-Control-Allow-Methods</name>
|
|
46
|
+
<value>POST, GET, OPTIONS</value>
|
|
47
|
+
</header>
|
|
48
|
+
<header>
|
|
49
|
+
<name>Access-Control-Max-Age</name>
|
|
50
|
+
<value>86400</value>
|
|
51
|
+
</header>
|
|
52
|
+
</http_options_response>
|
|
53
|
+
|
|
54
|
+
</clickhouse>
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<clickhouse>
|
|
3
|
+
|
|
4
|
+
<profiles>
|
|
5
|
+
<default>
|
|
6
|
+
<load_balancing>random</load_balancing>
|
|
7
|
+
</default>
|
|
8
|
+
</profiles>
|
|
9
|
+
|
|
10
|
+
<users>
|
|
11
|
+
<default>
|
|
12
|
+
<password></password>
|
|
13
|
+
<networks>
|
|
14
|
+
<ip>::/0</ip>
|
|
15
|
+
</networks>
|
|
16
|
+
<profile>default</profile>
|
|
17
|
+
<quota>default</quota>
|
|
18
|
+
<access_management>1</access_management>
|
|
19
|
+
</default>
|
|
20
|
+
</users>
|
|
21
|
+
|
|
22
|
+
<quotas>
|
|
23
|
+
<default>
|
|
24
|
+
<interval>
|
|
25
|
+
<duration>3600</duration>
|
|
26
|
+
<queries>0</queries>
|
|
27
|
+
<errors>0</errors>
|
|
28
|
+
<result_rows>0</result_rows>
|
|
29
|
+
<read_rows>0</read_rows>
|
|
30
|
+
<execution_time>0</execution_time>
|
|
31
|
+
</interval>
|
|
32
|
+
</default>
|
|
33
|
+
</quotas>
|
|
34
|
+
</clickhouse>
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
version: '3.5'
|
|
2
|
+
|
|
3
|
+
services:
|
|
4
|
+
clickhouse1:
|
|
5
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
|
6
|
+
ulimits:
|
|
7
|
+
nofile:
|
|
8
|
+
soft: 262144
|
|
9
|
+
hard: 262144
|
|
10
|
+
hostname: clickhouse1
|
|
11
|
+
container_name: clickhouse-activerecord-clickhouse-server-1
|
|
12
|
+
ports:
|
|
13
|
+
- '8124:8123'
|
|
14
|
+
- '9001:9000'
|
|
15
|
+
volumes:
|
|
16
|
+
- './clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml'
|
|
17
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
|
18
|
+
|
|
19
|
+
clickhouse2:
|
|
20
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
|
21
|
+
ulimits:
|
|
22
|
+
nofile:
|
|
23
|
+
soft: 262144
|
|
24
|
+
hard: 262144
|
|
25
|
+
hostname: clickhouse2
|
|
26
|
+
container_name: clickhouse-activerecord-clickhouse-server-2
|
|
27
|
+
ports:
|
|
28
|
+
- '8125:8123'
|
|
29
|
+
volumes:
|
|
30
|
+
- './clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml'
|
|
31
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
|
32
|
+
|
|
33
|
+
# Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests
|
|
34
|
+
nginx:
|
|
35
|
+
image: 'nginx:1.23.1-alpine'
|
|
36
|
+
hostname: nginx
|
|
37
|
+
ports:
|
|
38
|
+
- '28123:8123'
|
|
39
|
+
volumes:
|
|
40
|
+
- './nginx/local.conf:/etc/nginx/conf.d/local.conf'
|
|
41
|
+
container_name: clickhouse-activerecord-nginx
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
version: '3.8'
|
|
2
|
+
services:
|
|
3
|
+
clickhouse:
|
|
4
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
|
5
|
+
container_name: 'clickhouse-activerecord-clickhouse-server'
|
|
6
|
+
ports:
|
|
7
|
+
- '18123:8123'
|
|
8
|
+
ulimits:
|
|
9
|
+
nofile:
|
|
10
|
+
soft: 262144
|
|
11
|
+
hard: 262144
|
|
12
|
+
volumes:
|
|
13
|
+
- './clickhouse/single/config.xml:/etc/clickhouse-server/config.xml'
|
|
14
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
name: Testing
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ "master" ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ "master" ]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
tests_single:
|
|
11
|
+
name: Testing single server
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
|
|
14
|
+
env:
|
|
15
|
+
CLICKHOUSE_PORT: 18123
|
|
16
|
+
CLICKHOUSE_DATABASE: default
|
|
17
|
+
|
|
18
|
+
strategy:
|
|
19
|
+
fail-fast: true
|
|
20
|
+
max-parallel: 1
|
|
21
|
+
matrix:
|
|
22
|
+
ruby-version: [ '2.7', '3.0', '3.2' ]
|
|
23
|
+
clickhouse: [ '22.1' ]
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
|
|
28
|
+
- name: Start ClickHouse ${{ matrix.clickhouse }}
|
|
29
|
+
uses: isbang/compose-action@v1.5.1
|
|
30
|
+
env:
|
|
31
|
+
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
|
|
32
|
+
with:
|
|
33
|
+
compose-file: '.docker/docker-compose.yml'
|
|
34
|
+
down-flags: '--volumes'
|
|
35
|
+
|
|
36
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
|
37
|
+
uses: ruby/setup-ruby@v1
|
|
38
|
+
with:
|
|
39
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
40
|
+
bundler-cache: true
|
|
41
|
+
|
|
42
|
+
- run: bundle exec rspec spec/single
|
|
43
|
+
|
|
44
|
+
tests_cluster:
|
|
45
|
+
name: Testing cluster server
|
|
46
|
+
runs-on: ubuntu-latest
|
|
47
|
+
|
|
48
|
+
env:
|
|
49
|
+
CLICKHOUSE_PORT: 28123
|
|
50
|
+
CLICKHOUSE_DATABASE: default
|
|
51
|
+
CLICKHOUSE_CLUSTER: test_cluster
|
|
52
|
+
|
|
53
|
+
strategy:
|
|
54
|
+
fail-fast: true
|
|
55
|
+
max-parallel: 1
|
|
56
|
+
matrix:
|
|
57
|
+
ruby-version: [ '2.7', '3.0', '3.2' ]
|
|
58
|
+
clickhouse: [ '22.1' ]
|
|
59
|
+
|
|
60
|
+
steps:
|
|
61
|
+
- uses: actions/checkout@v4
|
|
62
|
+
|
|
63
|
+
- name: Start ClickHouse Cluster ${{ matrix.clickhouse }}
|
|
64
|
+
uses: isbang/compose-action@v1.5.1
|
|
65
|
+
env:
|
|
66
|
+
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
|
|
67
|
+
with:
|
|
68
|
+
compose-file: '.docker/docker-compose.cluster.yml'
|
|
69
|
+
down-flags: '--volumes'
|
|
70
|
+
|
|
71
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
|
72
|
+
uses: ruby/setup-ruby@v1
|
|
73
|
+
with:
|
|
74
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
75
|
+
bundler-cache: true
|
|
76
|
+
|
|
77
|
+
- run: bundle exec rspec spec/cluster
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,31 @@
|
|
|
1
|
+
### Version 1.0.5 (Mar 14, 2024)
|
|
2
|
+
|
|
3
|
+
* GitHub workflows
|
|
4
|
+
* Fix injection internal and schema classes for rails 7
|
|
5
|
+
* Add support for binary string by [@PauloMiranda98](https://github.com/PauloMiranda98) in (#116)
|
|
6
|
+
|
|
7
|
+
### Version 1.0.4 (Feb 2, 2024)
|
|
8
|
+
|
|
9
|
+
* Use ILIKE for `model.arel_table[:column]#matches` by [@stympy](https://github.com/stympy) in (#115)
|
|
10
|
+
* Fixed `insert_all` for array column (#71)
|
|
11
|
+
* Register Bool and UUID in type map by [@lukinski](https://github.com/lukinski) in (#110)
|
|
12
|
+
* Refactoring `final` method
|
|
13
|
+
* Support update & delete for clickhouse from version 23.3 and newer (#93)
|
|
14
|
+
|
|
15
|
+
### Version 1.0.0 (Nov 29, 2023)
|
|
16
|
+
|
|
17
|
+
* Full support Rails 7.1+
|
|
18
|
+
* Full support primary or multiple databases
|
|
19
|
+
|
|
20
|
+
### Version 0.6.0 (Oct 19, 2023)
|
|
21
|
+
|
|
22
|
+
* Added `Bool` column type instead `Uint8` (#78). Supports ClickHouse 22+ database only
|
|
23
|
+
* Added `final` method (#81) (The `ar_internal_metadata` table needs to be deleted after a gem update)
|
|
24
|
+
* Added `settings` method (#82)
|
|
25
|
+
* Fixed convert aggregation type (#92)
|
|
26
|
+
* Fixed raise error not database exist (#91)
|
|
27
|
+
* Fixed internal metadata update (#84)
|
|
28
|
+
|
|
1
29
|
### Version 0.5.10 (Jun 22, 2022)
|
|
2
30
|
|
|
3
31
|
* Fixes to create_table method (#70)
|
|
@@ -6,6 +6,8 @@ module ActiveRecord
|
|
|
6
6
|
module ConnectionAdapters
|
|
7
7
|
module Clickhouse
|
|
8
8
|
module SchemaStatements
|
|
9
|
+
DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
|
|
10
|
+
|
|
9
11
|
def execute(sql, name = nil, settings: {})
|
|
10
12
|
do_execute(sql, name, settings: settings)
|
|
11
13
|
end
|
|
@@ -33,13 +35,20 @@ module ActiveRecord
|
|
|
33
35
|
# @link https://clickhouse.com/docs/en/sql-reference/statements/alter/update
|
|
34
36
|
def exec_update(_sql, _name = nil, _binds = [])
|
|
35
37
|
do_execute(_sql, _name, format: nil)
|
|
36
|
-
|
|
38
|
+
0
|
|
37
39
|
end
|
|
38
40
|
|
|
39
41
|
# @link https://clickhouse.com/docs/en/sql-reference/statements/delete
|
|
40
42
|
def exec_delete(_sql, _name = nil, _binds = [])
|
|
41
|
-
|
|
42
|
-
|
|
43
|
+
log(_sql, "#{adapter_name} #{_name}") do
|
|
44
|
+
res = request(_sql)
|
|
45
|
+
begin
|
|
46
|
+
data = JSON.parse(res.header['x-clickhouse-summary'])
|
|
47
|
+
data['result_rows'].to_i
|
|
48
|
+
rescue JSONError
|
|
49
|
+
0
|
|
50
|
+
end
|
|
51
|
+
end
|
|
43
52
|
end
|
|
44
53
|
|
|
45
54
|
def tables(name = nil)
|
|
@@ -64,19 +73,15 @@ module ActiveRecord
|
|
|
64
73
|
|
|
65
74
|
def do_system_execute(sql, name = nil)
|
|
66
75
|
log_with_debug(sql, "#{adapter_name} #{name}") do
|
|
67
|
-
res =
|
|
68
|
-
|
|
69
|
-
process_response(res)
|
|
76
|
+
res = request(sql, DEFAULT_RESPONSE_FORMAT)
|
|
77
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
|
70
78
|
end
|
|
71
79
|
end
|
|
72
80
|
|
|
73
|
-
def do_execute(sql, name = nil, format:
|
|
81
|
+
def do_execute(sql, name = nil, format: DEFAULT_RESPONSE_FORMAT, settings: {})
|
|
74
82
|
log(sql, "#{adapter_name} #{name}") do
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
|
|
78
|
-
|
|
79
|
-
process_response(res)
|
|
83
|
+
res = request(sql, format, settings)
|
|
84
|
+
process_response(res, format)
|
|
80
85
|
end
|
|
81
86
|
end
|
|
82
87
|
|
|
@@ -96,7 +101,7 @@ module ActiveRecord
|
|
|
96
101
|
if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
|
|
97
102
|
raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
|
|
98
103
|
end
|
|
99
|
-
do_execute(insert_versions_sql(inserting), nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
|
|
104
|
+
do_execute(insert_versions_sql(inserting), nil, format: nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
|
|
100
105
|
end
|
|
101
106
|
end
|
|
102
107
|
|
|
@@ -112,17 +117,28 @@ module ActiveRecord
|
|
|
112
117
|
|
|
113
118
|
private
|
|
114
119
|
|
|
120
|
+
# Make HTTP request to ClickHouse server
|
|
121
|
+
# @param [String] sql
|
|
122
|
+
# @param [String, nil] format
|
|
123
|
+
# @param [Hash] settings
|
|
124
|
+
# @return [Net::HTTPResponse]
|
|
125
|
+
def request(sql, format = nil, settings = {})
|
|
126
|
+
formatted_sql = apply_format(sql, format)
|
|
127
|
+
request_params = @connection_config || {}
|
|
128
|
+
@connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
|
|
129
|
+
end
|
|
130
|
+
|
|
115
131
|
def apply_format(sql, format)
|
|
116
132
|
format ? "#{sql} FORMAT #{format}" : sql
|
|
117
133
|
end
|
|
118
134
|
|
|
119
|
-
def process_response(res)
|
|
135
|
+
def process_response(res, format)
|
|
120
136
|
case res.code.to_i
|
|
121
137
|
when 200
|
|
122
138
|
if res.body.to_s.include?("DB::Exception")
|
|
123
139
|
raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}"
|
|
124
140
|
else
|
|
125
|
-
res.body
|
|
141
|
+
format_body_response(res.body, format)
|
|
126
142
|
end
|
|
127
143
|
else
|
|
128
144
|
case res.body
|
|
@@ -168,8 +184,7 @@ module ActiveRecord
|
|
|
168
184
|
|
|
169
185
|
return data unless data.empty?
|
|
170
186
|
|
|
171
|
-
raise ActiveRecord::StatementInvalid,
|
|
172
|
-
"Could not find table '#{table_name}'"
|
|
187
|
+
raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'"
|
|
173
188
|
end
|
|
174
189
|
alias column_definitions table_structure
|
|
175
190
|
|
|
@@ -203,6 +218,40 @@ module ActiveRecord
|
|
|
203
218
|
def has_default_function?(default_value, default) # :nodoc:
|
|
204
219
|
!default_value && (%r{\w+\(.*\)} === default)
|
|
205
220
|
end
|
|
221
|
+
|
|
222
|
+
def format_body_response(body, format)
|
|
223
|
+
return body if body.blank?
|
|
224
|
+
|
|
225
|
+
case format
|
|
226
|
+
when 'JSONCompact'
|
|
227
|
+
format_from_json_compact(body)
|
|
228
|
+
when 'JSONCompactEachRowWithNamesAndTypes'
|
|
229
|
+
format_from_json_compact_each_row_with_names_and_types(body)
|
|
230
|
+
else
|
|
231
|
+
body
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def format_from_json_compact(body)
|
|
236
|
+
JSON.parse(body)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def format_from_json_compact_each_row_with_names_and_types(body)
|
|
240
|
+
rows = body.split("\n").map { |row| JSON.parse(row) }
|
|
241
|
+
names, types, *data = rows
|
|
242
|
+
|
|
243
|
+
meta = names.zip(types).map do |name, type|
|
|
244
|
+
{
|
|
245
|
+
'name' => name,
|
|
246
|
+
'type' => type
|
|
247
|
+
}
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
{
|
|
251
|
+
'meta' => meta,
|
|
252
|
+
'data' => data
|
|
253
|
+
}
|
|
254
|
+
end
|
|
206
255
|
end
|
|
207
256
|
end
|
|
208
257
|
end
|
|
@@ -73,10 +73,11 @@ module ActiveRecord
|
|
|
73
73
|
def is_view=(value)
|
|
74
74
|
@is_view = value
|
|
75
75
|
end
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
|
|
77
|
+
def _delete_record(constraints)
|
|
78
|
+
raise ActiveRecord::ActiveRecordError.new('Deleting a row is not possible without a primary key') unless self.primary_key
|
|
79
|
+
super
|
|
80
|
+
end
|
|
80
81
|
end
|
|
81
82
|
end
|
|
82
83
|
|
|
@@ -267,7 +268,7 @@ module ActiveRecord
|
|
|
267
268
|
sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
|
|
268
269
|
log_with_debug(sql, adapter_name) do
|
|
269
270
|
res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
|
|
270
|
-
process_response(res)
|
|
271
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
|
271
272
|
end
|
|
272
273
|
end
|
|
273
274
|
|
|
@@ -312,7 +313,7 @@ module ActiveRecord
|
|
|
312
313
|
sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
|
|
313
314
|
log_with_debug(sql, adapter_name) do
|
|
314
315
|
res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
|
|
315
|
-
process_response(res)
|
|
316
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
|
316
317
|
end
|
|
317
318
|
end
|
|
318
319
|
|
|
@@ -13,6 +13,13 @@ module Arel
|
|
|
13
13
|
end
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
+
# https://clickhouse.com/docs/en/sql-reference/statements/delete
|
|
17
|
+
# DELETE and UPDATE in ClickHouse working only without table name
|
|
18
|
+
def visit_Arel_Attributes_Attribute(o, collector)
|
|
19
|
+
collector << quote_table_name(o.relation.table_alias || o.relation.name) << '.' unless collector.value.start_with?('DELETE FROM ') || collector.value.include?(' UPDATE ')
|
|
20
|
+
collector << quote_column_name(o.name)
|
|
21
|
+
end
|
|
22
|
+
|
|
16
23
|
def visit_Arel_Nodes_SelectOptions(o, collector)
|
|
17
24
|
maybe_visit o.settings, super
|
|
18
25
|
end
|
|
@@ -24,9 +24,9 @@ end
|
|
|
24
24
|
|
|
25
25
|
module ClickhouseActiverecord
|
|
26
26
|
def self.load
|
|
27
|
-
ActiveRecord::InternalMetadata.
|
|
27
|
+
ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods)
|
|
28
28
|
ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation)
|
|
29
|
-
ActiveRecord::SchemaMigration.
|
|
29
|
+
ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods)
|
|
30
30
|
|
|
31
31
|
Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore)
|
|
32
32
|
Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement)
|
|
@@ -3,17 +3,6 @@ module CoreExtensions
|
|
|
3
3
|
module InternalMetadata
|
|
4
4
|
module ClassMethods
|
|
5
5
|
|
|
6
|
-
def []=(key, value)
|
|
7
|
-
row = final.find_by(key: key)
|
|
8
|
-
if row.nil? || row.value != value
|
|
9
|
-
create!(key: key, value: value)
|
|
10
|
-
end
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
def [](key)
|
|
14
|
-
final.where(key: key).pluck(:value).first
|
|
15
|
-
end
|
|
16
|
-
|
|
17
6
|
def create_table
|
|
18
7
|
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
19
8
|
return if table_exists? || !enabled?
|
|
@@ -21,7 +10,7 @@ module CoreExtensions
|
|
|
21
10
|
key_options = connection.internal_string_options_for_primary_key
|
|
22
11
|
table_options = {
|
|
23
12
|
id: false,
|
|
24
|
-
options:
|
|
13
|
+
options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key',
|
|
25
14
|
if_not_exists: true
|
|
26
15
|
}
|
|
27
16
|
full_config = connection.instance_variable_get(:@config) || {}
|
|
@@ -40,6 +29,27 @@ module CoreExtensions
|
|
|
40
29
|
t.timestamps
|
|
41
30
|
end
|
|
42
31
|
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def update_entry(key, new_value)
|
|
36
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
37
|
+
|
|
38
|
+
create_entry(key, new_value)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def select_entry(key)
|
|
42
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
43
|
+
|
|
44
|
+
sm = ::Arel::SelectManager.new(arel_table)
|
|
45
|
+
sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/
|
|
46
|
+
sm.project(::Arel.star)
|
|
47
|
+
sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key)))
|
|
48
|
+
sm.order(arel_table[primary_key].asc)
|
|
49
|
+
sm.limit = 1
|
|
50
|
+
|
|
51
|
+
connection.select_one(sm, "#{self.class} Load")
|
|
52
|
+
end
|
|
43
53
|
end
|
|
44
54
|
end
|
|
45
55
|
end
|
|
@@ -32,7 +32,7 @@ module CoreExtensions
|
|
|
32
32
|
def delete_version(version)
|
|
33
33
|
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
34
34
|
|
|
35
|
-
im = Arel::InsertManager.new(arel_table)
|
|
35
|
+
im = ::Arel::InsertManager.new(arel_table)
|
|
36
36
|
im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0)
|
|
37
37
|
connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version)
|
|
38
38
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: clickhouse-activerecord
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sergey Odintsov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-03-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -87,6 +87,14 @@ executables: []
|
|
|
87
87
|
extensions: []
|
|
88
88
|
extra_rdoc_files: []
|
|
89
89
|
files:
|
|
90
|
+
- ".docker/clickhouse/cluster/server1_config.xml"
|
|
91
|
+
- ".docker/clickhouse/cluster/server2_config.xml"
|
|
92
|
+
- ".docker/clickhouse/single/config.xml"
|
|
93
|
+
- ".docker/clickhouse/users.xml"
|
|
94
|
+
- ".docker/docker-compose.cluster.yml"
|
|
95
|
+
- ".docker/docker-compose.yml"
|
|
96
|
+
- ".docker/nginx/local.conf"
|
|
97
|
+
- ".github/workflows/testing.yml"
|
|
90
98
|
- ".gitignore"
|
|
91
99
|
- ".rspec"
|
|
92
100
|
- CHANGELOG.md
|