clickhouse-activerecord 1.0.4 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.docker/clickhouse/cluster/server1_config.xml +117 -0
- data/.docker/clickhouse/cluster/server2_config.xml +117 -0
- data/.docker/clickhouse/single/config.xml +54 -0
- data/.docker/clickhouse/users.xml +34 -0
- data/.docker/docker-compose.cluster.yml +41 -0
- data/.docker/docker-compose.yml +14 -0
- data/.docker/nginx/local.conf +12 -0
- data/.github/workflows/testing.yml +77 -0
- data/CHANGELOG.md +28 -0
- data/lib/active_record/connection_adapters/clickhouse/schema_statements.rb +66 -17
- data/lib/active_record/connection_adapters/clickhouse_adapter.rb +7 -6
- data/lib/arel/visitors/clickhouse.rb +7 -0
- data/lib/clickhouse-activerecord/version.rb +1 -1
- data/lib/clickhouse-activerecord.rb +2 -2
- data/lib/core_extensions/active_record/internal_metadata.rb +22 -12
- data/lib/core_extensions/active_record/schema_migration.rb +1 -1
- metadata +10 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2217c1539b22307398fbc9c01bb8b521300ca23a7d03c2d32a56a9abdd3c5546
|
|
4
|
+
data.tar.gz: 8aa60f1a78db38df4120b0b405ddaf9466da70e68bf382f58945b29d5e130b7b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d1e48e302475eff9ca738b35fc56b58a7a73f7d6b6cf5ae6632a7ee95fbc9432734ba18c53f7756d833ebd3748368fd73f7c29e04a0972f3141d00b5acd262cd
|
|
7
|
+
data.tar.gz: acedbb7298cd044769a5a1deaaae44672bdbf1affca5d2218e8b7209731df505434ed4f69bb23881e0d4f231e9e6699ac621ab0b4ac19f86475dd6c6515b3e18
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<clickhouse>
|
|
3
|
+
|
|
4
|
+
<http_port>8123</http_port>
|
|
5
|
+
<interserver_http_port>9009</interserver_http_port>
|
|
6
|
+
<interserver_http_host>clickhouse1</interserver_http_host>
|
|
7
|
+
|
|
8
|
+
<users_config>users.xml</users_config>
|
|
9
|
+
<default_profile>default</default_profile>
|
|
10
|
+
<default_database>default</default_database>
|
|
11
|
+
|
|
12
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
|
13
|
+
|
|
14
|
+
<path>/var/lib/clickhouse/</path>
|
|
15
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
16
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
17
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
|
18
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
|
19
|
+
|
|
20
|
+
<logger>
|
|
21
|
+
<level>debug</level>
|
|
22
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
|
23
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
|
24
|
+
<size>1000M</size>
|
|
25
|
+
<count>10</count>
|
|
26
|
+
<console>1</console>
|
|
27
|
+
</logger>
|
|
28
|
+
|
|
29
|
+
<remote_servers>
|
|
30
|
+
<test_cluster>
|
|
31
|
+
<shard>
|
|
32
|
+
<replica>
|
|
33
|
+
<host>clickhouse1</host>
|
|
34
|
+
<port>9000</port>
|
|
35
|
+
</replica>
|
|
36
|
+
<replica>
|
|
37
|
+
<host>clickhouse2</host>
|
|
38
|
+
<port>9000</port>
|
|
39
|
+
</replica>
|
|
40
|
+
</shard>
|
|
41
|
+
</test_cluster>
|
|
42
|
+
</remote_servers>
|
|
43
|
+
|
|
44
|
+
<keeper_server>
|
|
45
|
+
<tcp_port>9181</tcp_port>
|
|
46
|
+
<server_id>1</server_id>
|
|
47
|
+
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
|
48
|
+
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
|
49
|
+
|
|
50
|
+
<coordination_settings>
|
|
51
|
+
<operation_timeout_ms>10000</operation_timeout_ms>
|
|
52
|
+
<session_timeout_ms>30000</session_timeout_ms>
|
|
53
|
+
<raft_logs_level>trace</raft_logs_level>
|
|
54
|
+
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
|
|
55
|
+
</coordination_settings>
|
|
56
|
+
|
|
57
|
+
<raft_configuration>
|
|
58
|
+
<server>
|
|
59
|
+
<id>1</id>
|
|
60
|
+
<hostname>clickhouse1</hostname>
|
|
61
|
+
<port>9000</port>
|
|
62
|
+
</server>
|
|
63
|
+
<server>
|
|
64
|
+
<id>2</id>
|
|
65
|
+
<hostname>clickhouse2</hostname>
|
|
66
|
+
<port>9000</port>
|
|
67
|
+
</server>
|
|
68
|
+
</raft_configuration>
|
|
69
|
+
</keeper_server>
|
|
70
|
+
|
|
71
|
+
<zookeeper>
|
|
72
|
+
<node>
|
|
73
|
+
<host>clickhouse1</host>
|
|
74
|
+
<port>9181</port>
|
|
75
|
+
</node>
|
|
76
|
+
<node>
|
|
77
|
+
<host>clickhouse2</host>
|
|
78
|
+
<port>9181</port>
|
|
79
|
+
</node>
|
|
80
|
+
</zookeeper>
|
|
81
|
+
|
|
82
|
+
<macros>
|
|
83
|
+
<cluster>test_cluster</cluster>
|
|
84
|
+
<replica>clickhouse1</replica>
|
|
85
|
+
<shard>1</shard>
|
|
86
|
+
</macros>
|
|
87
|
+
|
|
88
|
+
<distributed_ddl>
|
|
89
|
+
<path>/clickhouse/test_cluster/task_queue/ddl</path>
|
|
90
|
+
</distributed_ddl>
|
|
91
|
+
|
|
92
|
+
<query_log>
|
|
93
|
+
<database>system</database>
|
|
94
|
+
<table>query_log</table>
|
|
95
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
|
96
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
|
97
|
+
</query_log>
|
|
98
|
+
|
|
99
|
+
<http_options_response>
|
|
100
|
+
<header>
|
|
101
|
+
<name>Access-Control-Allow-Origin</name>
|
|
102
|
+
<value>*</value>
|
|
103
|
+
</header>
|
|
104
|
+
<header>
|
|
105
|
+
<name>Access-Control-Allow-Headers</name>
|
|
106
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
|
107
|
+
</header>
|
|
108
|
+
<header>
|
|
109
|
+
<name>Access-Control-Allow-Methods</name>
|
|
110
|
+
<value>POST, GET, OPTIONS</value>
|
|
111
|
+
</header>
|
|
112
|
+
<header>
|
|
113
|
+
<name>Access-Control-Max-Age</name>
|
|
114
|
+
<value>86400</value>
|
|
115
|
+
</header>
|
|
116
|
+
</http_options_response>
|
|
117
|
+
</clickhouse>
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<clickhouse>
|
|
3
|
+
|
|
4
|
+
<http_port>8123</http_port>
|
|
5
|
+
<interserver_http_port>9009</interserver_http_port>
|
|
6
|
+
<interserver_http_host>clickhouse2</interserver_http_host>
|
|
7
|
+
|
|
8
|
+
<users_config>users.xml</users_config>
|
|
9
|
+
<default_profile>default</default_profile>
|
|
10
|
+
<default_database>default</default_database>
|
|
11
|
+
|
|
12
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
|
13
|
+
|
|
14
|
+
<path>/var/lib/clickhouse/</path>
|
|
15
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
16
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
17
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
|
18
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
|
19
|
+
|
|
20
|
+
<logger>
|
|
21
|
+
<level>debug</level>
|
|
22
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
|
23
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
|
24
|
+
<size>1000M</size>
|
|
25
|
+
<count>10</count>
|
|
26
|
+
<console>1</console>
|
|
27
|
+
</logger>
|
|
28
|
+
|
|
29
|
+
<remote_servers>
|
|
30
|
+
<test_cluster>
|
|
31
|
+
<shard>
|
|
32
|
+
<replica>
|
|
33
|
+
<host>clickhouse1</host>
|
|
34
|
+
<port>9000</port>
|
|
35
|
+
</replica>
|
|
36
|
+
<replica>
|
|
37
|
+
<host>clickhouse2</host>
|
|
38
|
+
<port>9000</port>
|
|
39
|
+
</replica>
|
|
40
|
+
</shard>
|
|
41
|
+
</test_cluster>
|
|
42
|
+
</remote_servers>
|
|
43
|
+
|
|
44
|
+
<keeper_server>
|
|
45
|
+
<tcp_port>9181</tcp_port>
|
|
46
|
+
<server_id>2</server_id>
|
|
47
|
+
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
|
48
|
+
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
|
49
|
+
|
|
50
|
+
<coordination_settings>
|
|
51
|
+
<operation_timeout_ms>10000</operation_timeout_ms>
|
|
52
|
+
<session_timeout_ms>30000</session_timeout_ms>
|
|
53
|
+
<raft_logs_level>trace</raft_logs_level>
|
|
54
|
+
<rotate_log_storage_interval>10000</rotate_log_storage_interval>
|
|
55
|
+
</coordination_settings>
|
|
56
|
+
|
|
57
|
+
<raft_configuration>
|
|
58
|
+
<server>
|
|
59
|
+
<id>1</id>
|
|
60
|
+
<hostname>clickhouse1</hostname>
|
|
61
|
+
<port>9000</port>
|
|
62
|
+
</server>
|
|
63
|
+
<server>
|
|
64
|
+
<id>2</id>
|
|
65
|
+
<hostname>clickhouse2</hostname>
|
|
66
|
+
<port>9000</port>
|
|
67
|
+
</server>
|
|
68
|
+
</raft_configuration>
|
|
69
|
+
</keeper_server>
|
|
70
|
+
|
|
71
|
+
<zookeeper>
|
|
72
|
+
<node>
|
|
73
|
+
<host>clickhouse1</host>
|
|
74
|
+
<port>9181</port>
|
|
75
|
+
</node>
|
|
76
|
+
<node>
|
|
77
|
+
<host>clickhouse2</host>
|
|
78
|
+
<port>9181</port>
|
|
79
|
+
</node>
|
|
80
|
+
</zookeeper>
|
|
81
|
+
|
|
82
|
+
<macros>
|
|
83
|
+
<cluster>test_cluster</cluster>
|
|
84
|
+
<replica>clickhouse2</replica>
|
|
85
|
+
<shard>1</shard>
|
|
86
|
+
</macros>
|
|
87
|
+
|
|
88
|
+
<distributed_ddl>
|
|
89
|
+
<path>/clickhouse/test_cluster/task_queue/ddl</path>
|
|
90
|
+
</distributed_ddl>
|
|
91
|
+
|
|
92
|
+
<query_log>
|
|
93
|
+
<database>system</database>
|
|
94
|
+
<table>query_log</table>
|
|
95
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
|
96
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
|
97
|
+
</query_log>
|
|
98
|
+
|
|
99
|
+
<http_options_response>
|
|
100
|
+
<header>
|
|
101
|
+
<name>Access-Control-Allow-Origin</name>
|
|
102
|
+
<value>*</value>
|
|
103
|
+
</header>
|
|
104
|
+
<header>
|
|
105
|
+
<name>Access-Control-Allow-Headers</name>
|
|
106
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
|
107
|
+
</header>
|
|
108
|
+
<header>
|
|
109
|
+
<name>Access-Control-Allow-Methods</name>
|
|
110
|
+
<value>POST, GET, OPTIONS</value>
|
|
111
|
+
</header>
|
|
112
|
+
<header>
|
|
113
|
+
<name>Access-Control-Max-Age</name>
|
|
114
|
+
<value>86400</value>
|
|
115
|
+
</header>
|
|
116
|
+
</http_options_response>
|
|
117
|
+
</clickhouse>
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<clickhouse>
|
|
3
|
+
|
|
4
|
+
<http_port>8123</http_port>
|
|
5
|
+
<tcp_port>9000</tcp_port>
|
|
6
|
+
|
|
7
|
+
<users_config>users.xml</users_config>
|
|
8
|
+
<default_profile>default</default_profile>
|
|
9
|
+
<default_database>default</default_database>
|
|
10
|
+
|
|
11
|
+
<mark_cache_size>5368709120</mark_cache_size>
|
|
12
|
+
|
|
13
|
+
<path>/var/lib/clickhouse/</path>
|
|
14
|
+
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
15
|
+
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
16
|
+
<access_control_path>/var/lib/clickhouse/access/</access_control_path>
|
|
17
|
+
<keep_alive_timeout>3</keep_alive_timeout>
|
|
18
|
+
|
|
19
|
+
<logger>
|
|
20
|
+
<level>debug</level>
|
|
21
|
+
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
|
22
|
+
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
|
23
|
+
<size>1000M</size>
|
|
24
|
+
<count>10</count>
|
|
25
|
+
<console>1</console>
|
|
26
|
+
</logger>
|
|
27
|
+
|
|
28
|
+
<query_log>
|
|
29
|
+
<database>system</database>
|
|
30
|
+
<table>query_log</table>
|
|
31
|
+
<partition_by>toYYYYMM(event_date)</partition_by>
|
|
32
|
+
<flush_interval_milliseconds>1000</flush_interval_milliseconds>
|
|
33
|
+
</query_log>
|
|
34
|
+
|
|
35
|
+
<http_options_response>
|
|
36
|
+
<header>
|
|
37
|
+
<name>Access-Control-Allow-Origin</name>
|
|
38
|
+
<value>*</value>
|
|
39
|
+
</header>
|
|
40
|
+
<header>
|
|
41
|
+
<name>Access-Control-Allow-Headers</name>
|
|
42
|
+
<value>accept, origin, x-requested-with, content-type, authorization</value>
|
|
43
|
+
</header>
|
|
44
|
+
<header>
|
|
45
|
+
<name>Access-Control-Allow-Methods</name>
|
|
46
|
+
<value>POST, GET, OPTIONS</value>
|
|
47
|
+
</header>
|
|
48
|
+
<header>
|
|
49
|
+
<name>Access-Control-Max-Age</name>
|
|
50
|
+
<value>86400</value>
|
|
51
|
+
</header>
|
|
52
|
+
</http_options_response>
|
|
53
|
+
|
|
54
|
+
</clickhouse>
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
<?xml version="1.0"?>
|
|
2
|
+
<clickhouse>
|
|
3
|
+
|
|
4
|
+
<profiles>
|
|
5
|
+
<default>
|
|
6
|
+
<load_balancing>random</load_balancing>
|
|
7
|
+
</default>
|
|
8
|
+
</profiles>
|
|
9
|
+
|
|
10
|
+
<users>
|
|
11
|
+
<default>
|
|
12
|
+
<password></password>
|
|
13
|
+
<networks>
|
|
14
|
+
<ip>::/0</ip>
|
|
15
|
+
</networks>
|
|
16
|
+
<profile>default</profile>
|
|
17
|
+
<quota>default</quota>
|
|
18
|
+
<access_management>1</access_management>
|
|
19
|
+
</default>
|
|
20
|
+
</users>
|
|
21
|
+
|
|
22
|
+
<quotas>
|
|
23
|
+
<default>
|
|
24
|
+
<interval>
|
|
25
|
+
<duration>3600</duration>
|
|
26
|
+
<queries>0</queries>
|
|
27
|
+
<errors>0</errors>
|
|
28
|
+
<result_rows>0</result_rows>
|
|
29
|
+
<read_rows>0</read_rows>
|
|
30
|
+
<execution_time>0</execution_time>
|
|
31
|
+
</interval>
|
|
32
|
+
</default>
|
|
33
|
+
</quotas>
|
|
34
|
+
</clickhouse>
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
version: '3.5'
|
|
2
|
+
|
|
3
|
+
services:
|
|
4
|
+
clickhouse1:
|
|
5
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
|
6
|
+
ulimits:
|
|
7
|
+
nofile:
|
|
8
|
+
soft: 262144
|
|
9
|
+
hard: 262144
|
|
10
|
+
hostname: clickhouse1
|
|
11
|
+
container_name: clickhouse-activerecord-clickhouse-server-1
|
|
12
|
+
ports:
|
|
13
|
+
- '8124:8123'
|
|
14
|
+
- '9001:9000'
|
|
15
|
+
volumes:
|
|
16
|
+
- './clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml'
|
|
17
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
|
18
|
+
|
|
19
|
+
clickhouse2:
|
|
20
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
|
21
|
+
ulimits:
|
|
22
|
+
nofile:
|
|
23
|
+
soft: 262144
|
|
24
|
+
hard: 262144
|
|
25
|
+
hostname: clickhouse2
|
|
26
|
+
container_name: clickhouse-activerecord-clickhouse-server-2
|
|
27
|
+
ports:
|
|
28
|
+
- '8125:8123'
|
|
29
|
+
volumes:
|
|
30
|
+
- './clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml'
|
|
31
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
|
32
|
+
|
|
33
|
+
# Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests
|
|
34
|
+
nginx:
|
|
35
|
+
image: 'nginx:1.23.1-alpine'
|
|
36
|
+
hostname: nginx
|
|
37
|
+
ports:
|
|
38
|
+
- '28123:8123'
|
|
39
|
+
volumes:
|
|
40
|
+
- './nginx/local.conf:/etc/nginx/conf.d/local.conf'
|
|
41
|
+
container_name: clickhouse-activerecord-nginx
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
version: '3.8'
|
|
2
|
+
services:
|
|
3
|
+
clickhouse:
|
|
4
|
+
image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-23.11-alpine}'
|
|
5
|
+
container_name: 'clickhouse-activerecord-clickhouse-server'
|
|
6
|
+
ports:
|
|
7
|
+
- '18123:8123'
|
|
8
|
+
ulimits:
|
|
9
|
+
nofile:
|
|
10
|
+
soft: 262144
|
|
11
|
+
hard: 262144
|
|
12
|
+
volumes:
|
|
13
|
+
- './clickhouse/single/config.xml:/etc/clickhouse-server/config.xml'
|
|
14
|
+
- './clickhouse/users.xml:/etc/clickhouse-server/users.xml'
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
name: Testing
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ "master" ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ "master" ]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
tests_single:
|
|
11
|
+
name: Testing single server
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
|
|
14
|
+
env:
|
|
15
|
+
CLICKHOUSE_PORT: 18123
|
|
16
|
+
CLICKHOUSE_DATABASE: default
|
|
17
|
+
|
|
18
|
+
strategy:
|
|
19
|
+
fail-fast: true
|
|
20
|
+
max-parallel: 1
|
|
21
|
+
matrix:
|
|
22
|
+
ruby-version: [ '2.7', '3.0', '3.2' ]
|
|
23
|
+
clickhouse: [ '22.1' ]
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
|
|
28
|
+
- name: Start ClickHouse ${{ matrix.clickhouse }}
|
|
29
|
+
uses: isbang/compose-action@v1.5.1
|
|
30
|
+
env:
|
|
31
|
+
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
|
|
32
|
+
with:
|
|
33
|
+
compose-file: '.docker/docker-compose.yml'
|
|
34
|
+
down-flags: '--volumes'
|
|
35
|
+
|
|
36
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
|
37
|
+
uses: ruby/setup-ruby@v1
|
|
38
|
+
with:
|
|
39
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
40
|
+
bundler-cache: true
|
|
41
|
+
|
|
42
|
+
- run: bundle exec rspec spec/single
|
|
43
|
+
|
|
44
|
+
tests_cluster:
|
|
45
|
+
name: Testing cluster server
|
|
46
|
+
runs-on: ubuntu-latest
|
|
47
|
+
|
|
48
|
+
env:
|
|
49
|
+
CLICKHOUSE_PORT: 28123
|
|
50
|
+
CLICKHOUSE_DATABASE: default
|
|
51
|
+
CLICKHOUSE_CLUSTER: test_cluster
|
|
52
|
+
|
|
53
|
+
strategy:
|
|
54
|
+
fail-fast: true
|
|
55
|
+
max-parallel: 1
|
|
56
|
+
matrix:
|
|
57
|
+
ruby-version: [ '2.7', '3.0', '3.2' ]
|
|
58
|
+
clickhouse: [ '22.1' ]
|
|
59
|
+
|
|
60
|
+
steps:
|
|
61
|
+
- uses: actions/checkout@v4
|
|
62
|
+
|
|
63
|
+
- name: Start ClickHouse Cluster ${{ matrix.clickhouse }}
|
|
64
|
+
uses: isbang/compose-action@v1.5.1
|
|
65
|
+
env:
|
|
66
|
+
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
|
|
67
|
+
with:
|
|
68
|
+
compose-file: '.docker/docker-compose.cluster.yml'
|
|
69
|
+
down-flags: '--volumes'
|
|
70
|
+
|
|
71
|
+
- name: Set up Ruby ${{ matrix.ruby-version }}
|
|
72
|
+
uses: ruby/setup-ruby@v1
|
|
73
|
+
with:
|
|
74
|
+
ruby-version: ${{ matrix.ruby-version }}
|
|
75
|
+
bundler-cache: true
|
|
76
|
+
|
|
77
|
+
- run: bundle exec rspec spec/cluster
|
data/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,31 @@
|
|
|
1
|
+
### Version 1.0.5 (Mar 14, 2024)
|
|
2
|
+
|
|
3
|
+
* GitHub workflows
|
|
4
|
+
* Fix injection internal and schema classes for rails 7
|
|
5
|
+
* Add support for binary string by [@PauloMiranda98](https://github.com/PauloMiranda98) in (#116)
|
|
6
|
+
|
|
7
|
+
### Version 1.0.4 (Feb 2, 2024)
|
|
8
|
+
|
|
9
|
+
* Use ILIKE for `model.arel_table[:column]#matches` by [@stympy](https://github.com/stympy) in (#115)
|
|
10
|
+
* Fixed `insert_all` for array column (#71)
|
|
11
|
+
* Register Bool and UUID in type map by [@lukinski](https://github.com/lukinski) in (#110)
|
|
12
|
+
* Refactoring `final` method
|
|
13
|
+
* Support update & delete for clickhouse from version 23.3 and newer (#93)
|
|
14
|
+
|
|
15
|
+
### Version 1.0.0 (Nov 29, 2023)
|
|
16
|
+
|
|
17
|
+
* Full support Rails 7.1+
|
|
18
|
+
* Full support primary or multiple databases
|
|
19
|
+
|
|
20
|
+
### Version 0.6.0 (Oct 19, 2023)
|
|
21
|
+
|
|
22
|
+
* Added `Bool` column type instead `Uint8` (#78). Supports ClickHouse 22+ database only
|
|
23
|
+
* Added `final` method (#81) (The `ar_internal_metadata` table needs to be deleted after a gem update)
|
|
24
|
+
* Added `settings` method (#82)
|
|
25
|
+
* Fixed convert aggregation type (#92)
|
|
26
|
+
* Fixed raise error not database exist (#91)
|
|
27
|
+
* Fixed internal metadata update (#84)
|
|
28
|
+
|
|
1
29
|
### Version 0.5.10 (Jun 22, 2022)
|
|
2
30
|
|
|
3
31
|
* Fixes to create_table method (#70)
|
|
@@ -6,6 +6,8 @@ module ActiveRecord
|
|
|
6
6
|
module ConnectionAdapters
|
|
7
7
|
module Clickhouse
|
|
8
8
|
module SchemaStatements
|
|
9
|
+
DEFAULT_RESPONSE_FORMAT = 'JSONCompactEachRowWithNamesAndTypes'.freeze
|
|
10
|
+
|
|
9
11
|
def execute(sql, name = nil, settings: {})
|
|
10
12
|
do_execute(sql, name, settings: settings)
|
|
11
13
|
end
|
|
@@ -33,13 +35,20 @@ module ActiveRecord
|
|
|
33
35
|
# @link https://clickhouse.com/docs/en/sql-reference/statements/alter/update
|
|
34
36
|
def exec_update(_sql, _name = nil, _binds = [])
|
|
35
37
|
do_execute(_sql, _name, format: nil)
|
|
36
|
-
|
|
38
|
+
0
|
|
37
39
|
end
|
|
38
40
|
|
|
39
41
|
# @link https://clickhouse.com/docs/en/sql-reference/statements/delete
|
|
40
42
|
def exec_delete(_sql, _name = nil, _binds = [])
|
|
41
|
-
|
|
42
|
-
|
|
43
|
+
log(_sql, "#{adapter_name} #{_name}") do
|
|
44
|
+
res = request(_sql)
|
|
45
|
+
begin
|
|
46
|
+
data = JSON.parse(res.header['x-clickhouse-summary'])
|
|
47
|
+
data['result_rows'].to_i
|
|
48
|
+
rescue JSONError
|
|
49
|
+
0
|
|
50
|
+
end
|
|
51
|
+
end
|
|
43
52
|
end
|
|
44
53
|
|
|
45
54
|
def tables(name = nil)
|
|
@@ -64,19 +73,15 @@ module ActiveRecord
|
|
|
64
73
|
|
|
65
74
|
def do_system_execute(sql, name = nil)
|
|
66
75
|
log_with_debug(sql, "#{adapter_name} #{name}") do
|
|
67
|
-
res =
|
|
68
|
-
|
|
69
|
-
process_response(res)
|
|
76
|
+
res = request(sql, DEFAULT_RESPONSE_FORMAT)
|
|
77
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
|
70
78
|
end
|
|
71
79
|
end
|
|
72
80
|
|
|
73
|
-
def do_execute(sql, name = nil, format:
|
|
81
|
+
def do_execute(sql, name = nil, format: DEFAULT_RESPONSE_FORMAT, settings: {})
|
|
74
82
|
log(sql, "#{adapter_name} #{name}") do
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
res = @connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
|
|
78
|
-
|
|
79
|
-
process_response(res)
|
|
83
|
+
res = request(sql, format, settings)
|
|
84
|
+
process_response(res, format)
|
|
80
85
|
end
|
|
81
86
|
end
|
|
82
87
|
|
|
@@ -96,7 +101,7 @@ module ActiveRecord
|
|
|
96
101
|
if (duplicate = inserting.detect { |v| inserting.count(v) > 1 })
|
|
97
102
|
raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict."
|
|
98
103
|
end
|
|
99
|
-
do_execute(insert_versions_sql(inserting), nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
|
|
104
|
+
do_execute(insert_versions_sql(inserting), nil, format: nil, settings: {max_partitions_per_insert_block: [100, inserting.size].max})
|
|
100
105
|
end
|
|
101
106
|
end
|
|
102
107
|
|
|
@@ -112,17 +117,28 @@ module ActiveRecord
|
|
|
112
117
|
|
|
113
118
|
private
|
|
114
119
|
|
|
120
|
+
# Make HTTP request to ClickHouse server
|
|
121
|
+
# @param [String] sql
|
|
122
|
+
# @param [String, nil] format
|
|
123
|
+
# @param [Hash] settings
|
|
124
|
+
# @return [Net::HTTPResponse]
|
|
125
|
+
def request(sql, format = nil, settings = {})
|
|
126
|
+
formatted_sql = apply_format(sql, format)
|
|
127
|
+
request_params = @connection_config || {}
|
|
128
|
+
@connection.post("/?#{request_params.merge(settings).to_param}", formatted_sql, 'User-Agent' => "Clickhouse ActiveRecord #{ClickhouseActiverecord::VERSION}")
|
|
129
|
+
end
|
|
130
|
+
|
|
115
131
|
def apply_format(sql, format)
|
|
116
132
|
format ? "#{sql} FORMAT #{format}" : sql
|
|
117
133
|
end
|
|
118
134
|
|
|
119
|
-
def process_response(res)
|
|
135
|
+
def process_response(res, format)
|
|
120
136
|
case res.code.to_i
|
|
121
137
|
when 200
|
|
122
138
|
if res.body.to_s.include?("DB::Exception")
|
|
123
139
|
raise ActiveRecord::ActiveRecordError, "Response code: #{res.code}:\n#{res.body}"
|
|
124
140
|
else
|
|
125
|
-
res.body
|
|
141
|
+
format_body_response(res.body, format)
|
|
126
142
|
end
|
|
127
143
|
else
|
|
128
144
|
case res.body
|
|
@@ -168,8 +184,7 @@ module ActiveRecord
|
|
|
168
184
|
|
|
169
185
|
return data unless data.empty?
|
|
170
186
|
|
|
171
|
-
raise ActiveRecord::StatementInvalid,
|
|
172
|
-
"Could not find table '#{table_name}'"
|
|
187
|
+
raise ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'"
|
|
173
188
|
end
|
|
174
189
|
alias column_definitions table_structure
|
|
175
190
|
|
|
@@ -203,6 +218,40 @@ module ActiveRecord
|
|
|
203
218
|
def has_default_function?(default_value, default) # :nodoc:
|
|
204
219
|
!default_value && (%r{\w+\(.*\)} === default)
|
|
205
220
|
end
|
|
221
|
+
|
|
222
|
+
def format_body_response(body, format)
|
|
223
|
+
return body if body.blank?
|
|
224
|
+
|
|
225
|
+
case format
|
|
226
|
+
when 'JSONCompact'
|
|
227
|
+
format_from_json_compact(body)
|
|
228
|
+
when 'JSONCompactEachRowWithNamesAndTypes'
|
|
229
|
+
format_from_json_compact_each_row_with_names_and_types(body)
|
|
230
|
+
else
|
|
231
|
+
body
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def format_from_json_compact(body)
|
|
236
|
+
JSON.parse(body)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def format_from_json_compact_each_row_with_names_and_types(body)
|
|
240
|
+
rows = body.split("\n").map { |row| JSON.parse(row) }
|
|
241
|
+
names, types, *data = rows
|
|
242
|
+
|
|
243
|
+
meta = names.zip(types).map do |name, type|
|
|
244
|
+
{
|
|
245
|
+
'name' => name,
|
|
246
|
+
'type' => type
|
|
247
|
+
}
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
{
|
|
251
|
+
'meta' => meta,
|
|
252
|
+
'data' => data
|
|
253
|
+
}
|
|
254
|
+
end
|
|
206
255
|
end
|
|
207
256
|
end
|
|
208
257
|
end
|
|
@@ -73,10 +73,11 @@ module ActiveRecord
|
|
|
73
73
|
def is_view=(value)
|
|
74
74
|
@is_view = value
|
|
75
75
|
end
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
|
|
77
|
+
def _delete_record(constraints)
|
|
78
|
+
raise ActiveRecord::ActiveRecordError.new('Deleting a row is not possible without a primary key') unless self.primary_key
|
|
79
|
+
super
|
|
80
|
+
end
|
|
80
81
|
end
|
|
81
82
|
end
|
|
82
83
|
|
|
@@ -267,7 +268,7 @@ module ActiveRecord
|
|
|
267
268
|
sql = apply_cluster "CREATE DATABASE #{quote_table_name(name)}"
|
|
268
269
|
log_with_debug(sql, adapter_name) do
|
|
269
270
|
res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
|
|
270
|
-
process_response(res)
|
|
271
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
|
271
272
|
end
|
|
272
273
|
end
|
|
273
274
|
|
|
@@ -312,7 +313,7 @@ module ActiveRecord
|
|
|
312
313
|
sql = apply_cluster "DROP DATABASE IF EXISTS #{quote_table_name(name)}"
|
|
313
314
|
log_with_debug(sql, adapter_name) do
|
|
314
315
|
res = @connection.post("/?#{@connection_config.except(:database).to_param}", sql)
|
|
315
|
-
process_response(res)
|
|
316
|
+
process_response(res, DEFAULT_RESPONSE_FORMAT)
|
|
316
317
|
end
|
|
317
318
|
end
|
|
318
319
|
|
|
@@ -13,6 +13,13 @@ module Arel
|
|
|
13
13
|
end
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
+
# https://clickhouse.com/docs/en/sql-reference/statements/delete
|
|
17
|
+
# DELETE and UPDATE in ClickHouse working only without table name
|
|
18
|
+
def visit_Arel_Attributes_Attribute(o, collector)
|
|
19
|
+
collector << quote_table_name(o.relation.table_alias || o.relation.name) << '.' unless collector.value.start_with?('DELETE FROM ') || collector.value.include?(' UPDATE ')
|
|
20
|
+
collector << quote_column_name(o.name)
|
|
21
|
+
end
|
|
22
|
+
|
|
16
23
|
def visit_Arel_Nodes_SelectOptions(o, collector)
|
|
17
24
|
maybe_visit o.settings, super
|
|
18
25
|
end
|
|
@@ -24,9 +24,9 @@ end
|
|
|
24
24
|
|
|
25
25
|
module ClickhouseActiverecord
|
|
26
26
|
def self.load
|
|
27
|
-
ActiveRecord::InternalMetadata.
|
|
27
|
+
ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods)
|
|
28
28
|
ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation)
|
|
29
|
-
ActiveRecord::SchemaMigration.
|
|
29
|
+
ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods)
|
|
30
30
|
|
|
31
31
|
Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore)
|
|
32
32
|
Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement)
|
|
@@ -3,17 +3,6 @@ module CoreExtensions
|
|
|
3
3
|
module InternalMetadata
|
|
4
4
|
module ClassMethods
|
|
5
5
|
|
|
6
|
-
def []=(key, value)
|
|
7
|
-
row = final.find_by(key: key)
|
|
8
|
-
if row.nil? || row.value != value
|
|
9
|
-
create!(key: key, value: value)
|
|
10
|
-
end
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
def [](key)
|
|
14
|
-
final.where(key: key).pluck(:value).first
|
|
15
|
-
end
|
|
16
|
-
|
|
17
6
|
def create_table
|
|
18
7
|
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
19
8
|
return if table_exists? || !enabled?
|
|
@@ -21,7 +10,7 @@ module CoreExtensions
|
|
|
21
10
|
key_options = connection.internal_string_options_for_primary_key
|
|
22
11
|
table_options = {
|
|
23
12
|
id: false,
|
|
24
|
-
options:
|
|
13
|
+
options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key',
|
|
25
14
|
if_not_exists: true
|
|
26
15
|
}
|
|
27
16
|
full_config = connection.instance_variable_get(:@config) || {}
|
|
@@ -40,6 +29,27 @@ module CoreExtensions
|
|
|
40
29
|
t.timestamps
|
|
41
30
|
end
|
|
42
31
|
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def update_entry(key, new_value)
|
|
36
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
37
|
+
|
|
38
|
+
create_entry(key, new_value)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def select_entry(key)
|
|
42
|
+
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
43
|
+
|
|
44
|
+
sm = ::Arel::SelectManager.new(arel_table)
|
|
45
|
+
sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/
|
|
46
|
+
sm.project(::Arel.star)
|
|
47
|
+
sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key)))
|
|
48
|
+
sm.order(arel_table[primary_key].asc)
|
|
49
|
+
sm.limit = 1
|
|
50
|
+
|
|
51
|
+
connection.select_one(sm, "#{self.class} Load")
|
|
52
|
+
end
|
|
43
53
|
end
|
|
44
54
|
end
|
|
45
55
|
end
|
|
@@ -32,7 +32,7 @@ module CoreExtensions
|
|
|
32
32
|
def delete_version(version)
|
|
33
33
|
return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
|
|
34
34
|
|
|
35
|
-
im = Arel::InsertManager.new(arel_table)
|
|
35
|
+
im = ::Arel::InsertManager.new(arel_table)
|
|
36
36
|
im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0)
|
|
37
37
|
connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version)
|
|
38
38
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: clickhouse-activerecord
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sergey Odintsov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-03-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -87,6 +87,14 @@ executables: []
|
|
|
87
87
|
extensions: []
|
|
88
88
|
extra_rdoc_files: []
|
|
89
89
|
files:
|
|
90
|
+
- ".docker/clickhouse/cluster/server1_config.xml"
|
|
91
|
+
- ".docker/clickhouse/cluster/server2_config.xml"
|
|
92
|
+
- ".docker/clickhouse/single/config.xml"
|
|
93
|
+
- ".docker/clickhouse/users.xml"
|
|
94
|
+
- ".docker/docker-compose.cluster.yml"
|
|
95
|
+
- ".docker/docker-compose.yml"
|
|
96
|
+
- ".docker/nginx/local.conf"
|
|
97
|
+
- ".github/workflows/testing.yml"
|
|
90
98
|
- ".gitignore"
|
|
91
99
|
- ".rspec"
|
|
92
100
|
- CHANGELOG.md
|