umbrellio-utils 1.9.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7aa36379cad4a44799273f1c8d85d305a1cbad842cdf6a54d41fc747e2166212
-  data.tar.gz: 98b110e358259114978e5f5507e0a2b781563e570a0185e8af9ed73e8ac706ce
+  metadata.gz: 2a7a306b7e789f1b3b24ce3ca8a94094f6d09a671f478df61b7e2cad5f66d402
+  data.tar.gz: 1f301027fd3863925d541841c35cd5bffa514c18cce59abfdb20576c0bba1fdd
 SHA512:
-  metadata.gz: 4bae7e5de101aa61d9261163699379f0fffdb5242c789dd80b1c55c46508cd836a8ce568584e00e2a36e3d954a11f6a349e5caa1909cfff426cfca07ae77f10a
-  data.tar.gz: 2e4432dc497061588cff9f4daf759f45fe330fdf866809295b14c6368bd50b4f010ec92a2588f7d6c44369ed7e30347466e06496f78f9aa1b24b1315515b027c
+  metadata.gz: 76b7400d1927214aca02810314e072ddcde0fd15c47b8becc0b0cae7ffd11f71ec3495ce07cf0d3ae346829faa87fc3f1d8fdb31cc9635a636ba49ede1312bac
+  data.tar.gz: 023dbebdfdaef806d5dbb952f24d2dad321d5c6d42abc05965811ba9f0d7499457626858099626f10b924f97a5918c3d14f43d4760f1e64fcebc9a774188ec8d

data/.github/clickhouse/clickhouse_keeper.xml

@@ -0,0 +1,42 @@
+<yandex>
+  <remote_servers>
+    <click_cluster>
+      <shard>
+        <internal_replication>true</internal_replication>
+        <replica>
+          <host>localhost</host>
+          <port>9000</port>
+        </replica>
+      </shard>
+    </click_cluster>
+  </remote_servers>
+
+  <keeper_server>
+    <tcp_port>9181</tcp_port>
+    <tcp_port_secure>0</tcp_port_secure>
+    <server_id>1</server_id>
+    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+    <coordination_settings>
+      <operation_timeout_ms>10000</operation_timeout_ms>
+      <session_timeout_ms>30000</session_timeout_ms>
+      <raft_logs_level>trace</raft_logs_level>
+    </coordination_settings>
+
+    <raft_configuration>
+      <server>
+        <id>1</id>
+        <hostname>localhost</hostname>
+        <port>9234</port>
+      </server>
+    </raft_configuration>
+  </keeper_server>
+
+  <zookeeper>
+    <node>
+      <host>localhost</host>
+      <port>9181</port>
+    </node>
+  </zookeeper>
+</yandex>

data/.github/workflows/test.yml

@@ -17,25 +17,52 @@ jobs:
       matrix:
         ruby: ["3.1", "3.2", "3.3", "3.4"]
 
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_USER: root
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
     env:
-      PGHOST: localhost
-      PGUSER: root
-
+      PGHOST: pg
+      PGUSER: user
+      PGPASSWORD: pass
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
+      - name: Create docker network
+        run: docker network create dbnet
+
+      - name: Start PostgreSQL
+        run: |
+          docker run -d \
+            --name pg \
+            --network dbnet \
+            -e POSTGRES_PASSWORD=pass \
+            -e POSTGRES_USER=user \
+            -e POSTGRES_DB=umbrellio_utils_test \
+            -p 5432:5432 \
+            postgres:14
+
+      - name: Start ClickHouse
+        run: |
+          docker run -d \
+            --name ch \
+            --network dbnet \
+            -e CLICKHOUSE_SKIP_USER_SETUP=1 -e CLICKHOUSE_DB=umbrellio_utils_test \
+            -p 9000:9000 -p 8123:8123 \
+            -v ${{ github.workspace }}/.github/clickhouse/clickhouse_keeper.xml:/etc/clickhouse-server/config.d/keeper.xml \
+            clickhouse/clickhouse-server:25.3.6.56-alpine
+
+      - name: Wait for Postgres
+        run: |
+          for i in {1..30}; do
+            if docker exec pg pg_isready -U user; then exit 0; fi
+            sleep 1
+          done
+          exit 1
+
+      - name: Wait for ClickHouse
+        run: |
+          for i in {1..30}; do
+            if docker exec ch clickhouse-client --query "SELECT 1"; then exit 0; fi
+            sleep 1
+          done
+          exit 1
 
       - uses: ruby/setup-ruby@v1
         with:
@@ -43,8 +70,6 @@ jobs:
           rubygems: latest
           bundler-cache: true
 
-      - run: psql -c 'CREATE DATABASE umbrellio_utils_test'
-
       - name: Run Linter
         run: bundle exec ci-helper RubocopLint
 

data/.rubocop.yml

@@ -12,6 +12,9 @@ Naming/MethodParameterName:
 RSpec/EmptyLineAfterHook:
   Enabled: false
 
+Metrics/ModuleLength:
+  Enabled: false
+
 Naming/FileName:
   Exclude:
     - lib/umbrellio-utils.rb

data/Gemfile

@@ -8,6 +8,8 @@ gemspec
 gem "activesupport"
 gem "bundler"
 gem "ci-helper"
+gem "click_house", github: "umbrellio/click_house", branch: "master"
+gem "csv"
 gem "http"
 gem "net-pop"
 gem "nokogiri"

data/Gemfile.lock

@@ -1,7 +1,16 @@
+GIT
+  remote: https://github.com/umbrellio/click_house.git
+  revision: 1bbf8cb909a248b401d0ba9a9f6f1de2e2c068db
+  branch: master
+  specs:
+    click_house (2.1.2)
+      activesupport
+      faraday (>= 1.7, < 3)
+
 PATH
   remote: .
   specs:
-    umbrellio-utils (1.9.0)
+    umbrellio-utils (1.10.0)
       memery (~> 1)
 
 GEM
@@ -100,6 +109,7 @@ GEM
     concurrent-ruby (1.3.5)
     connection_pool (2.5.4)
     crass (1.0.6)
+    csv (3.3.5)
     date (3.4.1)
     diff-lcs (1.6.2)
     docile (1.4.1)
@@ -109,6 +119,12 @@ GEM
     erb (4.0.4)
       cgi (>= 0.3.3)
     erubi (1.13.1)
+    faraday (2.14.0)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.2)
+      net-http (~> 0.5)
     ffi (1.17.2-aarch64-linux-gnu)
     ffi (1.17.2-aarch64-linux-musl)
     ffi (1.17.2-arm-linux-gnu)
@@ -163,6 +179,8 @@ GEM
     method_source (1.1.0)
     mini_mime (1.1.5)
     minitest (5.25.5)
+    net-http (0.8.0)
+      uri (>= 0.11.1)
     net-imap (0.5.10)
       date
       net-protocol
@@ -364,6 +382,7 @@ GEM
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.1.0)
+    uri (1.1.1)
     useragent (0.16.11)
     websocket-driver (0.8.0)
       base64
@@ -388,6 +407,8 @@ DEPENDENCIES
   activesupport
   bundler
   ci-helper
+  click_house!
+  csv
   http
   net-pop
   nokogiri

data/bin/clickhouse-server

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -eu
+
+docker stop clickhouse-server || true
+docker rm clickhouse-server || true
+
+docker run \
+  --detach \
+  --network host \
+  --name clickhouse-server \
+  --ulimit nofile=262144:262144 \
+  $CLICKHOUSE_IMAGE_TAG
+
+# Wait for ClickHouse server to become available
+until docker exec clickhouse-server clickhouse-client --query "SELECT 1" &>/dev/null; do
+  echo "Waiting for ClickHouse to be ready..."
+  sleep 1
+done
+
+rails ch:create ch:migrate

data/lib/umbrellio-utils.rb

@@ -1,3 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "umbrellio_utils"
+
+if defined?(Rake)
+  Dir[File.join(__dir__, "umbrellio_utils/tasks/**/*.rake")].each { |f| load f }
+end

data/lib/umbrellio_utils/click_house.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module UmbrellioUtils
+  module ClickHouse
+    include Memery
+
+    extend self
+
+    delegate :create_database, :drop_database, :tables, :config, to: :client
+
+    def insert(table_name, db_name: self.db_name, rows: [])
+      client.insert(full_table_name(table_name, db_name), rows, format: "JSONEachRow")
+    end
+
+    def from(source, db_name: self.db_name)
+      ds =
+        case source
+        when Symbol
+          DB.from(db_name == self.db_name ? SQL[source] : SQL[db_name][source])
+        when nil
+          DB.dataset
+        else
+          DB.from(source)
+        end
+
+      ds.clone(ch: true)
+    end
+
+    def execute(sql, host: nil, **opts)
+      log_errors(sql) do
+        client(host).execute(sql, params: opts)
+      end
+    end
+
+    def query(dataset, host: nil, **opts)
+      sql = sql_for(dataset)
+
+      log_errors(sql) do
+        select_all(sql, host:, **opts).map { |x| Misc::StrictHash[x.symbolize_keys] }
+      end
+    end
+
+    def query_value(dataset, host: nil, **opts)
+      sql = sql_for(dataset)
+
+      log_errors(sql) do
+        select_value(sql, host:, **opts)
+      end
+    end
+
+    def count(dataset)
+      query_value(dataset.select(SQL.ch_count))
+    end
+
+    def optimize_table!(table_name, db_name: self.db_name)
+      execute("OPTIMIZE TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster FINAL")
+    end
+
+    def truncate_table!(table_name, db_name: self.db_name)
+      execute("TRUNCATE TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster SYNC")
+    end
+
+    def drop_table!(table_name, db_name: self.db_name)
+      execute("DROP TABLE #{db_name}.#{table_name} ON CLUSTER click_cluster SYNC")
+    end
+
+    def describe_table(table_name, db_name: self.db_name)
+      sql = "DESCRIBE TABLE #{full_table_name(table_name, db_name)} FORMAT JSON"
+
+      log_errors(sql) do
+        select_all(sql).map { |x| Misc::StrictHash[x.symbolize_keys] }
+      end
+    end
+
+    def db_name
+      client.config.database.to_sym
+    end
+
+    def parse_value(value, type:)
+      case type
+      when /String/
+        value&.to_s
+      when /DateTime/
+        Time.zone.parse(value) if value
+      else
+        value
+      end
+    end
+
+    def server_version
+      select_value("SELECT version()").to_f
+    end
+
+    def pg_table_connection(table)
+      host = ENV["PGHOST"] || DB.opts[:host].presence || "localhost"
+      port = DB.opts[:port] || 5432
+      database = DB.opts[:database]
+      username = DB.opts[:user]
+      password = DB.opts[:password]
+
+      Sequel.function(:postgresql, "#{host}:#{port}", database, table, username, password)
+    end
+
+    def with_temp_table(
+      dataset, temp_table_name:, primary_key: [:id], primary_key_types: [:integer], **opts, &
+    )
+      unless DB.table_exists?(temp_table_name)
+        UmbrellioUtils::Database.create_temp_table(
+          nil, primary_key:, primary_key_types:, temp_table_name:, &
+        )
+        populate_temp_table!(temp_table_name, dataset)
+      end
+      UmbrellioUtils::Database.with_temp_table(nil, primary_key:, temp_table_name:, **opts, &)
+    end
+
+    private
+
+    def client(host = nil)
+      cfg = ::ClickHouse.config
+      cfg.host = resolve(host) if host
+      ::ClickHouse::Connection.new(cfg)
+    end
+    memoize :client, ttl: 1.minute
+
+    def resolve(host)
+      IPSocket.getaddress(host)
+    rescue => e
+      Exceptions.notify!(e, raise_errors: false)
+      config.host
+    end
+
+    def logger
+      client.config.logger
+    end
+
+    def log_errors(sql)
+      yield
+    rescue ::ClickHouse::Error => e
+      logger.error("ClickHouse error: #{e.inspect}\nSQL: #{sql}")
+      raise e
+    end
+
+    def sql_for(dataset)
+      unless ch_dataset?(dataset)
+        raise "Non-ClickHouse dataset: #{dataset.inspect}. " \
+              "You should use `CH.from` instead of `DB`"
+      end
+
+      dataset.sql
+    end
+
+    def ch_dataset?(dataset)
+      case dataset
+      when Sequel::Dataset
+        dataset.opts[:ch] && Array(dataset.opts[:from]).all? { |x| ch_dataset?(x) }
+      when Sequel::SQL::AliasedExpression
+        ch_dataset?(dataset.expression)
+      when Sequel::SQL::Identifier, Sequel::SQL::QualifiedIdentifier
+        true
+      else
+        raise "Unknown dataset type: #{dataset.inspect}"
+      end
+    end
+
+    def full_table_name(table_name, db_name)
+      table_name = table_name.value if table_name.is_a?(Sequel::SQL::Identifier)
+      "#{db_name}.#{table_name}"
+    end
+
+    def select_all(sql, host: nil, **opts)
+      response = client(host).get(body: sql, query: { default_format: "JSON", **opts })
+      ::ClickHouse::Response::Factory.response(response, client(host).config)
+    end
+
+    def select_value(...)
+      select_all(...).first.to_a.dig(0, -1)
+    end
+
+    def populate_temp_table!(temp_table_name, dataset)
+      execute(<<~SQL.squish)
+        INSERT INTO TABLE FUNCTION #{DB.literal(pg_table_connection(temp_table_name))}
+        #{dataset.sql}
+      SQL
+    end
+  end
+end

data/lib/umbrellio_utils/database.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# rubocop:disable Metrics/ModuleLength
 module UmbrellioUtils
   module Database
     extend self
@@ -79,26 +78,28 @@ module UmbrellioUtils
     end
     # rubocop:enable Metrics/ParameterLists
 
-    def create_temp_table(dataset, primary_key: nil, temp_table_name: nil)
+    def create_temp_table(dataset, primary_key: nil, primary_key_types: nil, temp_table_name: nil)
       time = Time.current
-      model = dataset.model
+      query_table_name = dataset&.model&.table_name
 
-      temp_table_name ||= :"temp_#{model.table_name}_#{time.to_i}_#{time.nsec}"
+      temp_table_name ||= :"temp_#{query_table_name}_#{time.to_i}_#{time.nsec}"
       return temp_table_name if DB.table_exists?(temp_table_name)
 
       primary_key = primary_key_from(dataset, primary_key:)
+      primary_key_types ||= primary_key.map { |x| dataset.model.db_schema[x][:db_type] }
 
       DB.create_table(temp_table_name, unlogged: true) do
-        primary_key.each do |field|
-          type = model.db_schema[field][:db_type]
-          column(field, type)
+        primary_key.each.with_index do |field, i|
+          column(field, primary_key_types[i])
         end
 
         primary_key(primary_key)
       end
 
-      insert_ds = dataset.select(*qualified_pk(model.table_name, primary_key))
-      DB[temp_table_name].disable_insert_returning.insert(insert_ds)
+      unless dataset.nil?
+        insert_ds = dataset.select(*qualified_pk(query_table_name, primary_key))
+        DB[temp_table_name].disable_insert_returning.insert(insert_ds)
+      end
 
       temp_table_name
     end
@@ -154,4 +155,3 @@ module UmbrellioUtils
     end
   end
 end
-# rubocop:enable Metrics/ModuleLength

data/lib/umbrellio_utils/migrations.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+module UmbrellioUtils
+  module Migrations
+    extend self
+
+    def create_new_id_bigint_column(table_name)
+      DB.run(<<~SQL.squish)
+        LOCK TABLE #{table_name} IN ACCESS EXCLUSIVE MODE;
+
+        CREATE OR REPLACE FUNCTION id_trigger()
+        RETURNS trigger
+        AS
+        $BODY$
+        DECLARE
+        BEGIN
+            NEW.id_bigint := NEW.id;
+            RETURN NEW;
+        END;
+        $BODY$ LANGUAGE plpgsql;
+
+        ALTER TABLE #{table_name} ADD id_bigint BIGINT;
+
+        CREATE TRIGGER #{table_name}_bigint
+          BEFORE INSERT OR UPDATE
+          ON #{table_name}
+          FOR EACH ROW
+          EXECUTE FUNCTION id_trigger();
+      SQL
+    end
+
+    def drop_old_id_column(table_name, associations = {}, skip_fk_create: false) # rubocop:disable Metrics/MethodLength
+      query_start = <<~SQL.squish
+        LOCK TABLE #{table_name} IN ACCESS EXCLUSIVE MODE;
+        DROP TRIGGER #{table_name}_bigint ON #{table_name};
+        ALTER TABLE #{table_name} RENAME id TO id_integer;
+        ALTER TABLE #{table_name} RENAME id_bigint TO id;
+
+        CREATE SEQUENCE IF NOT EXISTS new_#{table_name}_id_seq
+          START WITH 1
+          INCREMENT BY 1
+          NO MINVALUE
+          NO MAXVALUE
+          CACHE 1;
+
+        SELECT setval(
+          'new_#{table_name}_id_seq',
+          COALESCE((SELECT MAX(id) + 1 FROM #{table_name}), 1),
+          false
+        );
+        ALTER TABLE #{table_name}
+          ALTER COLUMN id SET DEFAULT nextval('new_#{table_name}_id_seq');
+      SQL
+
+      fkey_query = ""
+      associations.map do |assoc_table, assoc_name|
+        constraint_name = "#{assoc_table}_#{assoc_name}_fkey"
+
+        fkey_query += <<~SQL.squish
+          ALTER TABLE #{assoc_table}
+          DROP CONSTRAINT IF EXISTS #{constraint_name}
+        SQL
+        if skip_fk_create
+          fkey_query += ";"
+          next
+        end
+
+        fkey_query += <<~SQL.squish
+          , ADD CONSTRAINT #{constraint_name}
+          FOREIGN KEY (#{assoc_name}) REFERENCES #{table_name}(id) NOT VALID;
+        SQL
+      end
+
+      query_end = <<~SQL.squish
+        ALTER TABLE #{table_name} DROP id_integer;
+        ALTER TABLE #{table_name} ADD CONSTRAINT #{table_name}_pkey PRIMARY KEY
+          USING INDEX #{table_name}_id_bigint_index;
+      SQL
+
+      query = query_start + fkey_query + query_end
+      DB.run(query)
+    end
+
+    def drop_foreign_keys(_table_name, associations)
+      associations.map do |assoc_table, assoc_name|
+        constraint_name = "#{assoc_table}_#{assoc_name}_fkey"
+        fkey_query = <<~SQL.squish
+          ALTER TABLE #{assoc_table} DROP CONSTRAINT IF EXISTS #{constraint_name};
+        SQL
+        DB.run(fkey_query)
+      end
+    end
+
+    def create_foreign_keys(table_name, associations)
+      associations.map do |assoc_table, assoc_name|
+        constraint_name = "#{assoc_table}_#{assoc_name}_fkey"
+        fkey_query = <<~SQL.squish
+          DO $$
+          BEGIN
+            IF NOT EXISTS (
+              SELECT 1
+              FROM pg_constraint
+              WHERE conname = '#{constraint_name}'
+            ) THEN
+              ALTER TABLE #{assoc_table} ADD CONSTRAINT #{constraint_name}
+                FOREIGN KEY (#{assoc_name}) REFERENCES #{table_name}(id) NOT VALID;
+            END IF;
+          END$$;
+        SQL
+        DB.run(fkey_query)
+      end
+    end
+
+    def create_new_foreign_key_column(table_name, column_name)
+      DB.run(<<~SQL.squish)
+        LOCK TABLE #{table_name} IN ACCESS EXCLUSIVE MODE;
+
+        CREATE OR REPLACE FUNCTION #{column_name}_bigint_trigger()
+        RETURNS trigger
+        AS
+        $BODY$
+        DECLARE
+        BEGIN
+            NEW.#{column_name}_bigint := NEW.#{column_name};
+            RETURN NEW;
+        END;
+        $BODY$ LANGUAGE plpgsql;
+
+        ALTER TABLE #{table_name} ADD #{column_name}_bigint BIGINT;
+
+        CREATE TRIGGER #{table_name}_#{column_name}_bigint
+          BEFORE INSERT OR UPDATE
+          ON #{table_name}
+          FOR EACH ROW
+          EXECUTE FUNCTION #{column_name}_bigint_trigger();
+      SQL
+    end
+
+    def check_id_consistency(table_name, col_name = "id")
+      res = DB[table_name].where(
+        Sequel[col_name.to_sym] !~ SQL.coalesce(Sequel[:"#{col_name}_bigint"], 0),
+      ).count
+      raise "Inconsistent ids in #{table_name}: #{res} records" if res.positive?
+      true
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def drop_old_foreign_key_column(table_name, column_name, skip_constraint: false,
+                                    primary_key: [], uniq_constr: false)
+      query_start = <<~SQL.squish
+        LOCK TABLE #{table_name} IN ACCESS EXCLUSIVE MODE;
+        DROP TRIGGER #{table_name}_#{column_name}_bigint ON #{table_name};
+        ALTER TABLE #{table_name} RENAME #{column_name} TO #{column_name}_integer;
+        ALTER TABLE #{table_name} RENAME #{column_name}_bigint TO #{column_name};
+      SQL
+
+      fkey_query = ""
+      unless skip_constraint
+        constraint_name = "#{table_name}_#{column_name}_fkey"
+        ref_table_name = column_name.to_s.delete_suffix("_id").pluralize
+        fkey_query = <<~SQL.squish
+          ALTER TABLE #{table_name}
+          DROP CONSTRAINT IF EXISTS #{constraint_name},
+          ADD CONSTRAINT #{constraint_name}
+          FOREIGN KEY (#{column_name}) REFERENCES #{ref_table_name}(id) NOT VALID;
+        SQL
+      end
+
+      drop_query = <<~SQL.squish
+        ALTER TABLE #{table_name} DROP #{column_name}_integer;
+      SQL
+
+      constr_query = ""
+      if uniq_constr
+        constr_query = <<~SQL.squish
+          ALTER TABLE #{table_name}
+          ADD CONSTRAINT #{table_name}_#{column_name}_key UNIQUE (#{column_name});
+        SQL
+      end
+
+      pkey_query = ""
+      if primary_key.present?
+        pkey_query = <<~SQL.squish
+          ALTER TABLE #{table_name} ADD CONSTRAINT #{table_name}_pkey PRIMARY KEY
+            USING INDEX #{table_name}_#{primary_key.join("_")}_index;
+        SQL
+      end
+
+      query = query_start + fkey_query + drop_query + constr_query + pkey_query
+      DB.run(query)
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def check_associations(model, method, reverse_method)
+      model.dataset.limit(10).all.each do |record|
+        res = record.public_send(method).public_send(reverse_method)
+        raise StandardError if res.blank?
+      end
+      true
+    end
+
+    def create_distributed_table!(table_name, sharding_key, db_name: UmbrellioUtils::ClickHouse.db_name)
+      UmbrellioUtils::ClickHouse.execute(<<~SQL.squish)
+        DROP TABLE IF EXISTS #{db_name}.#{table_name}_distributed
+        ON CLUSTER click_cluster
+      SQL
+
+      UmbrellioUtils::ClickHouse.execute(<<~SQL.squish)
+        CREATE TABLE #{db_name}.#{table_name}_distributed
+        ON CLUSTER click_cluster
+        AS #{db_name}.#{table_name}
+        ENGINE = Distributed(click_cluster, #{db_name}, #{table_name}, #{sharding_key})
+      SQL
+    end
+
+    # @example
+    # add_columns_to_view(
+    #   "orders_clickhouse_view",
+    #   Sequel[:orders][:data].pg_jsonb.get_text("some_data_column").as(:some_column),
+    #   Sequel[:orders][:column].as(:some_other_column),
+    # )
+    def add_columns_to_view(view_name, *sequel_columns)
+      sequel_columns.each do |column|
+        unless column.is_a?(Sequel::SQL::AliasedExpression)
+          raise ArgumentError.new("not Sequel::SQL::AliasedExpression")
+        end
+      end
+
+      DB.transaction do
+        DB.run("LOCK TABLE #{view_name}")
+        definition = view_definition(view_name)
+        sql = sequel_columns.map { |x| DB.literal(x) }.join(", ")
+        new_definition = definition.sub("FROM", ", #{sql} FROM")
+        DB.run("CREATE OR REPLACE VIEW #{view_name} AS #{new_definition}")
+      end
+    end
+
+    # @example
+    # drop_columns_from_view("orders_clickhouse_view", "id", "guid")
+    def drop_columns_from_view(view_name, *columns)
+      DB.transaction do
+        DB.run("LOCK TABLE #{view_name}")
+        definition = view_definition(view_name)
+        parsed_columns = parse_columns(definition)
+        parsed_columns.reject! { |name, _| name.in?(columns) }
+        sql = parsed_columns.map { |_, sql| sql }.join(", ")
+        new_definition = definition.sub(/SELECT(.*?)FROM/i, "SELECT #{sql} FROM")
+        DB.run("DROP VIEW #{view_name}")
+        DB.run("CREATE VIEW #{view_name} AS #{new_definition}")
+      end
+    end
+
+    private
+
+    def parse_columns(definition)
+      fields_sql = definition[/SELECT(.*?)FROM/i, 1].strip
+      fields = fields_sql.scan(/(?:[^,(]+|\([^)]*\))+/).map(&:strip)
+      field_names = fields.map do |field|
+        field[/as (.*)/i, 1] || field[/\.(.*)\z/, 1]
+      end
+      field_names.zip(fields)
+    end
+
+    def view_definition(view)
+      DB[:pg_views]
+        .where(viewname: view.to_s)
+        .select(:definition).first[:definition]
+        .squish
+    end
+  end
+end

data/lib/umbrellio_utils/misc.rb

@@ -4,6 +4,11 @@ module UmbrellioUtils
   module Misc
     extend self
 
+    class StrictHash < Hash
+      alias get []
+      alias [] fetch
+    end
+
     def table_sync(scope, delay: 1, routing_key: nil)
       scope.in_batches do |batch|
         batch_for_sync = batch.all.reject { |model| model.try(:skip_table_sync?) }

data/lib/umbrellio_utils/sql.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+module UmbrellioUtils
+  module SQL
+    extend self
+
+    UniqueConstraintViolation = Sequel::UniqueConstraintViolation
+
+    def [](*args)
+      Sequel[*args]
+    end
+
+    def func(...)
+      Sequel.function(...)
+    end
+
+    def cast(...)
+      Sequel.cast(...)
+    end
+
+    def case(...)
+      Sequel.case(...)
+    end
+
+    def pg_jsonb(...)
+      Sequel.pg_jsonb(...)
+    end
+
+    def to_utc(date)
+      func(:timezone, "UTC", date)
+    end
+
+    def to_timezone(zone, date)
+      utc_date = to_utc(date)
+      func(:timezone, zone, cast(utc_date, :timestamptz))
+    end
+
+    def and(*conditions)
+      Sequel.&(*Array(conditions.flatten.presence || true))
+    end
+
+    def not(...)
+      Sequel.~(...)
+    end
+
+    def or(*conditions)
+      Sequel.|(*Array(conditions.flatten.presence || true))
+    end
+
+    def pg_range(from_value, to_value, **opts)
+      Sequel::Postgres::PGRange.new(from_value, to_value, **opts)
+    end
+
+    def pg_range_by_range(range)
+      Sequel::Postgres::PGRange.from_range(range)
+    end
+
+    def max(expr)
+      func(:max, expr)
+    end
+
+    def min(expr)
+      func(:min, expr)
+    end
+
+    def sum(expr)
+      func(:sum, expr)
+    end
+
+    def count(expr = nil)
+      expr ? func(:count, expr) : func(:count).*
+    end
+
+    def ch_count(*args)
+      Sequel.function(:count, *args)
+    end
+
+    def avg(expr)
+      func(:avg, expr)
+    end
+
+    def pg_percentile(expr, percentile)
+      func(:percentile_cont, percentile).within_group(expr)
+    end
+
+    def pg_median(expr)
+      pg_percentile(expr, 0.5)
+    end
+
+    def ch_median(expr)
+      func(:median, expr)
+    end
+
+    def abs(expr)
+      func(:abs, expr)
+    end
+
+    def coalesce(*exprs)
+      func(:coalesce, *exprs)
+    end
+
+    def coalesce0(*args)
+      coalesce(*args, 0)
+    end
+
+    def nullif(main_expr, checking_expr)
+      func(:nullif, main_expr, checking_expr)
+    end
+
+    def distinct(expr)
+      func(:distinct, expr)
+    end
+
+    def least(*exprs)
+      func(:least, *exprs)
+    end
+
+    def greatest(*exprs)
+      func(:greatest, *exprs)
+    end
+
+    def date_trunc(truncate, expr)
+      func(:date_trunc, truncate.to_s, expr)
+    end
+
+    def ch_timestamp(time)
+      time&.strftime("%F %T.%6N")
+    end
+
+    def ch_timestamp_expr(time)
+      time = Time.zone.parse(time) if time.is_a?(String)
+      func(:toDateTime64, Sequel[ch_timestamp(time)], 6)
+    end
+
+    def ch_time_range(range)
+      Range.new(ch_timestamp(range.begin), ch_timestamp(range.end), range.exclude_end?)
+    end
+
+    def jsonb_dig(jsonb, path)
+      path.reduce(jsonb) { |acc, cur| acc[cur] }
+    end
+
+    def jsonb_typeof(jsonb)
+      func(:jsonb_typeof, jsonb)
+    end
+
+    def empty_jsonb
+      Sequel.pg_jsonb({})
+    end
+
+    def round(value, precision = 0)
+      func(:round, value, precision)
+    end
+
+    def row(*values)
+      func(:row, *values)
+    end
+
+    def map_to_expr(hash)
+      hash.map { |aliaz, expr| expr.as(aliaz) }
+    end
+
+    def intersect(left_expr, right_expr)
+      Sequel.lit("SELECT ? INTERSECT SELECT ?", left_expr, right_expr)
+    end
+
+    # can rewrite scalar values
+    def jsonb_unsafe_set(jsonb, path, value)
+      parent_path = path.slice(..-2)
+      raw_parent = jsonb_dig(jsonb, parent_path)
+      parent = jsonb_rewrite_scalar(raw_parent)
+      last_path = path.slice(-1..-1)
+      updated_parent = parent.set(last_path, value)
+      result = self.case({ { value => nil } => parent }, updated_parent)
+      jsonb.set(parent_path, result)
+    end
+
+    def true
+      Sequel.lit("true")
+    end
+
+    def false
+      Sequel.lit("false")
+    end
+
+    private
+
+    def jsonb_rewrite_scalar(jsonb)
+      self.case({ { jsonb_typeof(jsonb) => %w[object array] } => jsonb }, empty_jsonb).pg_jsonb
+    end
+  end
+end

data/lib/umbrellio_utils/tasks/clickhouse_connect.rake

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+namespace :ch do
+  desc "run clickhouse client"
+  task connect: :environment do
+    params = {
+      host: ENV.fetch("CLICKHOUSE_HOST", UmbrellioUtils::ClickHouse.config.host),
+      user: ENV.fetch("CLICKHOUSE_USER", UmbrellioUtils::ClickHouse.config.username),
+      password: ENV.fetch("CLICKHOUSE_PASSWORD", UmbrellioUtils::ClickHouse.config.password),
+      database: ENV.fetch("CLICKHOUSE_DATABASE", UmbrellioUtils::ClickHouse.config.database),
+      **UmbrellioUtils::ClickHouse.config.global_params,
+    }.compact_blank
+
+    cmd = Shellwords.join(["clickhouse", "client", *params.map { |k, v| "--#{k}=#{v}" }])
+    exec(cmd)
+  end
+end

data/lib/umbrellio_utils/testing/sequel_patches.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    # Make Postgres return rows truly randomly in specs unless order is properly specified
+    class Sequel::Postgres::Dataset # rubocop:disable Lint/ConstantDefinitionInBlock
+      def select_sql
+        return super if @opts[:_skip_order_patch] || @opts[:append_sql]
+        return super if @opts[:ch] && @opts[:order].present?
+        order = @opts[:order].dup || []
+        fn = @opts.key?(:ch) ? :rand : :random
+        order << Sequel.function(fn)
+        clone(order:, _skip_order_patch: true).select_sql
+      end
+    end
+  end
+end

data/lib/umbrellio_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module UmbrellioUtils
-  VERSION = "1.9.0"
+  VERSION = "1.10.0"
 end

data/lib/umbrellio_utils.rb

@@ -48,18 +48,21 @@ end
 
 require_relative "umbrellio_utils/cards"
 require_relative "umbrellio_utils/checks"
+require_relative "umbrellio_utils/click_house"
 require_relative "umbrellio_utils/constants"
 require_relative "umbrellio_utils/control"
 require_relative "umbrellio_utils/database"
 require_relative "umbrellio_utils/formatting"
 require_relative "umbrellio_utils/http_client"
 require_relative "umbrellio_utils/jobs"
+require_relative "umbrellio_utils/migrations"
 require_relative "umbrellio_utils/misc"
 require_relative "umbrellio_utils/parsing"
 require_relative "umbrellio_utils/passwords"
 require_relative "umbrellio_utils/random"
 require_relative "umbrellio_utils/request_wrapper"
 require_relative "umbrellio_utils/rounding"
+require_relative "umbrellio_utils/sql"
 require_relative "umbrellio_utils/semantic_logger/tiny_json_formatter"
 require_relative "umbrellio_utils/store"
 require_relative "umbrellio_utils/vault"

data/umbrellio_utils.gemspec

@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = "bin"
+  spec.executables   = ["clickhouse-server"]
   spec.require_paths = ["lib"]
 
   spec.add_dependency "memery", "~> 1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: umbrellio-utils
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Team Umbrellio
 autorequire:
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2025-10-29 00:00:00.000000000 Z
+date: 2025-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: memery
@@ -27,11 +27,13 @@ dependencies:
 description: UmbrellioUtils is collection of utility classes and helpers
 email:
 - oss@umbrellio.biz
-executables: []
+executables:
+- clickhouse-server
 extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
+- ".github/clickhouse/clickhouse_keeper.xml"
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
@@ -41,18 +43,21 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/clickhouse-server
 - bin/console
 - bin/setup
 - lib/umbrellio-utils.rb
 - lib/umbrellio_utils.rb
 - lib/umbrellio_utils/cards.rb
 - lib/umbrellio_utils/checks.rb
+- lib/umbrellio_utils/click_house.rb
 - lib/umbrellio_utils/constants.rb
 - lib/umbrellio_utils/control.rb
 - lib/umbrellio_utils/database.rb
 - lib/umbrellio_utils/formatting.rb
 - lib/umbrellio_utils/http_client.rb
 - lib/umbrellio_utils/jobs.rb
+- lib/umbrellio_utils/migrations.rb
 - lib/umbrellio_utils/misc.rb
 - lib/umbrellio_utils/parsing.rb
 - lib/umbrellio_utils/passwords.rb
@@ -60,7 +65,10 @@ files:
 - lib/umbrellio_utils/request_wrapper.rb
 - lib/umbrellio_utils/rounding.rb
 - lib/umbrellio_utils/semantic_logger/tiny_json_formatter.rb
+- lib/umbrellio_utils/sql.rb
 - lib/umbrellio_utils/store.rb
+- lib/umbrellio_utils/tasks/clickhouse_connect.rake
+- lib/umbrellio_utils/testing/sequel_patches.rb
 - lib/umbrellio_utils/vault.rb
 - lib/umbrellio_utils/version.rb
 - log/.keep