click_house 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 05f0f9426500a803ec5d2d37b1a2a06d2eb2d3e4b934eac19a434acc5086dc5c
+  data.tar.gz: c079c080dedbcc3342bbfde789099ddacf714a16029779bddbd8352d1912f1b0
+SHA512:
+  metadata.gz: f2f1d0d8f438b317ee475499e24f9d3ecae9f2b6a872a21695f11e0f8d744693790cd6f1caa50beab7e2c11f008952e453ca37d9282caeed18a95f54e99dff44
+  data.tar.gz: 17f7aff9dad97b3ad4384618b8cf8ce92f550ec9ee5b7ee524413458dce48f7eb08a69d4f78abf5862ac01acd33f1257de37403f6ce42595b152435819bc2e2f

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/pkg/
+/spec/reports/
+/log/*
+/tmp/*
+
+!/log/.keep
+!/tmp/.keep
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,125 @@
+require:
+  - rubocop-performance
+
+AllCops:
+  AutoCorrect: false
+  Exclude:
+    - 'click_house.gemspec'
+    - 'bin/*'
+    - 'spec/**/*'
+    - 'vendor/**/*'
+  TargetRubyVersion: 2.6.5
+
+Bundler/OrderedGems:
+  Enabled: false
+
+# ============================== Documentation ======================
+Style/Documentation:
+  Enabled: false
+
+# ============================== Metrics ============================
+Metrics/ClassLength:
+  Max: 180
+Metrics/BlockLength:
+  Enabled: true
+Metrics/MethodLength:
+  Max: 25
+Metrics/LineLength:
+  Max: 140
+Metrics/AbcSize:
+  Max: 40
+
+# ============================== Naming =============================
+Naming/PredicateName:
+  NamePrefixBlacklist:
+    - is_
+Naming/FileName:
+  Enabled: true
+  Exclude:
+    - 'Gemfile'
+Naming/UncommunicativeMethodParamName:
+  Enabled: false
+Naming/AccessorMethodName:
+  Enabled: false
+
+# ============================== Layout =============================
+Layout/AlignHash:
+  EnforcedHashRocketStyle: key
+  EnforcedColonStyle: key
+Layout/AlignParameters:
+  EnforcedStyle: with_fixed_indentation
+Layout/CaseIndentation:
+  EnforcedStyle: case
+  IndentOneStep: false
+Layout/MultilineMethodCallIndentation:
+  Enabled: true
+  EnforcedStyle: indented
+Layout/SpaceBeforeBlockBraces:
+  EnforcedStyle: space
+  EnforcedStyleForEmptyBraces: space
+Layout/EmptyLines:
+  Enabled: true
+Layout/EmptyLineAfterMagicComment:
+  Enabled: false
+Layout/EmptyLinesAroundBlockBody:
+  Enabled: true
+Layout/EndAlignment:
+  EnforcedStyleAlignWith: variable
+Layout/IndentFirstHashElement:
+  EnforcedStyle: consistent
+Layout/IndentHeredoc:
+  Enabled: false
+Layout/RescueEnsureAlignment:
+  Enabled: false
+
+# ============================== Style ==============================
+Style/RescueModifier:
+  Enabled: true
+Style/PercentLiteralDelimiters:
+  PreferredDelimiters:
+    default: '[]'
+    '%i':    '[]'
+    '%w':    '[]'
+  Exclude:
+    - 'config/routes.rb'
+Style/StringLiterals:
+  Enabled: true
+Style/AsciiComments:
+  Enabled: false
+Style/Copyright:
+  Enabled: false
+Style/SafeNavigation:
+  Enabled: false
+Style/Lambda:
+  Enabled: false
+Style/Alias:
+  Enabled: true
+  EnforcedStyle: prefer_alias_method
+Style/ClassAndModuleChildren:
+  Enabled: true
+  EnforcedStyle: nested
+Style/TrailingCommaInArrayLiteral:
+  Enabled: true
+  EnforcedStyleForMultiline: no_comma
+Style/RescueStandardError:
+  Enabled: true
+  EnforcedStyle: explicit
+Style/InverseMethods:
+  AutoCorrect: false
+  Enabled: true
+Style/IfUnlessModifier:
+  Enabled: false
+Style/SpecialGlobalVars:
+  Enabled: false
+Style/BlockComments:
+  Enabled: false
+Style/GuardClause:
+  Enabled: false
+Style/TrailingCommaInHashLiteral:
+  Enabled: false
+
+# ============================== Lint ==============================
+Lint/DuplicateMethods:
+  Enabled: false
+Lint/AmbiguousOperator:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+
+rvm:
+  - 2.6.5
+
+services:
+  - docker
+
+before_install:
+  - docker pull yandex/clickhouse-server
+  - docker run -d -p 127.0.0.1:8123:8123 yandex/clickhouse-server
+  - bundle install
+
+script:
+  - bundle exec rubocop
+  - bundle exec rspec

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in click_house.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: .
+  specs:
+    click_house (1.0.0)
+      faraday
+      faraday_middleware
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    coderay (1.1.2)
+    diff-lcs (1.3)
+    faraday (0.17.0)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.13.1)
+      faraday (>= 0.7.4, < 1.0)
+    jaro_winkler (1.5.4)
+    method_source (0.9.2)
+    multipart-post (2.1.1)
+    parallel (1.18.0)
+    parser (2.6.5.0)
+      ast (~> 2.4.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rainbow (3.0.0)
+    rake (13.0.0)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
+    rubocop (0.76.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    rubocop-performance (1.5.0)
+      rubocop (>= 0.71.0)
+    ruby-progressbar (1.10.1)
+    unicode-display_width (1.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.17)
+  click_house!
+  pry
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rubocop
+  rubocop-performance
+
+BUNDLED WITH
+   1.17.3

data/Makefile

@@ -0,0 +1,9 @@
+.PHONY: help
+
+help:
+	@echo 'Available targets:'
+	@echo '  make dockerize OR make ARGS="--build" dockerize'
+
+dockerize:
+	rm -f tmp/pids/server.pid tmp/sockets/puma.sock
+	docker-compose up ${ARGS}

data/README.md

@@ -0,0 +1,413 @@
+![](./doc/logo.svg?sanitize=true)
+
+# ClickHouse Ruby driver
+
+[![pipeline status](https://travis-ci.com/shlima/click_house.svg?branch=master)](https://travis-ci.com/shlima/click_house)
+
+```bash
+gem install click_house
+```
+
+A modern Ruby database driver for ClickHouse. [ClickHouse](https://clickhouse.yandex) 
+is a high-performance column-oriented database management system developed by 
+[Yandex](https://yandex.com/company) which operates Russia's most popular search engine.
+
+> This development was inspired by currently [unmaintainable alternative](https://github.com/archan937/clickhouse)
+> but rewritten and well tested. Requires modern Ruby (>= 2.6) and Yandex ClickHouse
+
+### Why use the HTTP interface and not the TCP interface?
+
+Well, the developers of ClickHouse themselves [discourage](https://github.com/yandex/ClickHouse/issues/45#issuecomment-231194134) using the TCP interface.
+
+> TCP transport is more specific, we don't want to expose details.
+Despite we have full compatibility of protocol of different versions of client and server, we want to keep the ability to "break" it for very old clients. And that protocol is not too clean to make a specification.
+
+# TOC
+
+* [Configuration](#configuration)
+* [Usage](#usage)
+* [Queries](#queries)
+* [Insert](#insert)
+* [Create a table](#create-a-table)
+* [Type casting](#type-casting)
+* [Using with a connection pool](#using-with-a-connection-pool)
+* [Using with Rails](#using-with-rails)
+
+## Configuration
+
+```ruby
+ClickHouse.config do |config|
+  config.logger = Logger.new(STDOUT)
+  config.adapter = :net_http
+  config.database = 'metrics'
+  config.url = 'http://localhost:8123'
+  config.timeout = 60
+  config.open_timeout = 3
+  config.ssl_verify = false
+  
+  # or provide connection options separately
+  config.scheme = 'http' 
+  config.host = 'localhost' 
+  config.port = 'port' 
+  
+  # if you use HTTP basic Auth
+  config.username = 'user' 
+  config.password = 'password' 
+end
+```
+
+Alternative, you can assign configuration parameters via a hash
+
+```ruby
+ClickHouse.config.assign(logger: Logger.new(STDOUT))
+```
+
+Now you are able to communicate with ClickHouse:
+
+```ruby
+ClickHouse.connection.ping #=> true
+```
+You can easily build a new raw connection
+
+```ruby
+@connection = ClickHouse::Connection.new(ClickHouse::Config.new(logger: Rails.logger))
+@connection.ping
+```
+
+## Usage
+
+```ruby
+ClickHouse.connection.ping #=> true
+ClickHouse.connection.replicas_status #=> true
+
+ClickHouse.connection.databases #=> ["default", "system"]
+ClickHouse.connection.create_database('metrics', if_not_exists: true, engine: nil, cluster: nil)
+ClickHouse.connection.drop_database('metrics', if_exists: true, cluster: nil)
+
+ClickHouse.connection.tables #=> ["visits"]
+ClickHouse.connection.describe_table('visits') #=> [{"name"=>"id", "type"=>"FixedString(16)", "default_type"=>""}]
+ClickHouse.connection.table_exists?('visits', temporary: nil) #=> true
+ClickHouse.connection.drop_table('visits', if_exists: true, temporary: nil, cluster: nil)
+ClickHouse.connection.create_table(*) # see <Create a table> section
+```
+
+## Queries
+### Select All
+
+Select all type-casted result set
+
+```ruby
+@result = ClickHouse.connection.select_all('SELECT * FROM visits')
+
+# all enumerable methods are delegated like #each, #map, #select etc
+@result.to_a #=> [{"date"=>#<Date: 2000-01-01>, "id"=>1}]
+
+# you can access raw data
+@result.meta #=> [{"name"=>"date", "type"=>"Date"}, {"name"=>"id", "type"=>"UInt32"}] 
+@result.data #=> [{"date"=>"2000-01-01", "id"=>1}, {"date"=>"2000-01-02", "id"=>2}] 
+@result.statistics #=> {"elapsed"=>0.0002271, "rows_read"=>2, "bytes_read"=>12}
+```
+
+### Select Value
+
+Select value returns exactly one type-casted value
+
+```ruby
+ClickHouse.connection.select_value('SELECT COUNT(*) from visits') #=> 0
+ClickHouse.connection.select_value("SELECT toDate('2019-01-01')") #=> #<Date: 2019-01-01>
+ClickHouse.connection.select_value("SELECT toDateOrZero(NULL)") #=> nil
+```
+
+### Select One
+
+Returns a record hash with the column names as keys and column values as values.
+
+```ruby
+ClickHouse.connection.select_one('SELECT date, SUM(id) AS sum FROM visits GROUP BY date')
+#=> {"date"=>#<Date: 2000-01-01>, "sum"=>1}
+```
+
+### Execute Raw SQL
+
+By default, gem provides parser for `JSON` and `CSV` response formats. Type conversion
+available for the `JSON`.
+
+```ruby
+# format not specified
+response = ClickHouse.connection.execute <<~SQL
+  SELECT count(*) AS counter FROM rspec
+SQL
+
+response.body #=> "2\n"
+
+# JSON
+response = ClickHouse.connection.execute <<~SQL
+  SELECT count(*) AS counter FROM rspec FORMAT JSON
+SQL
+
+response.body #=> {"meta"=>[{"name"=>"counter", "type"=>"UInt64"}], "data"=>[{"counter"=>"2"}], "rows"=>1, "statistics"=>{"elapsed"=>0.0002412, "rows_read"=>2, "bytes_read"=>4}}
+
+# CSV
+response = ClickHouse.connection.execute <<~SQL
+  SELECT count(*) AS counter FROM rspec FORMAT CSV
+SQL
+
+response.body #=> [["2"]]
+
+# You may use any format supported by ClickHouse
+response = ClickHouse.connection.execute <<~SQL
+  SELECT count(*) AS counter FROM rspec FORMAT RowBinary
+SQL
+
+response.body #=> "\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000" 
+```
+
+## Insert
+
+```ruby
+ClickHouse.connection.insert('table', columns: %i[id name]) do |buffer|
+  buffer << [1, 'Mercury']
+  buffer << [2, 'Venus']
+end 
+
+ClickHouse.connection.insert('table', columns: %i[id name], values: [[1, 'Mercury'], [2, 'Venus']])
+#=> true
+```
+
+## Create a table
+### Create table using DSL
+
+```ruby
+ClickHouse.connection.create_table('visits', if_not_exists: true, engine: 'MergeTree(date, (year, date), 8192)') do |t|
+  t.FixedString :id, 16
+  t.UInt16      :year
+  t.Date        :date
+  t.DateTime    :time, 'UTC'
+  t.Decimal     :money, 5, 4
+  t.String      :event
+  t.UInt32      :user_id
+  t.UInt32      :Float32
+end
+```
+
+### Create nullable columns
+
+```ruby
+ClickHouse.connection.create_table('visits', engine: 'TinyLog') do |t|
+  t.UInt16 :id, 16, nullable: true
+end
+```
+
+### Set column options
+
+```ruby
+ClickHouse.connection.create_table('visits', engine: 'MergeTree(date, (year, date), 8192)') do |t|
+  t.UInt16  :year
+  t.Date    :date
+  t.UInt16  :id, 16, default: 0, ttl: 'date + INTERVAL 1 DAY'
+end
+```
+
+### Define column with custom SQL
+
+```ruby
+ClickHouse.connection.create_table('visits', engine: 'TinyLog') do |t|
+  t << "vendor Enum('microsoft' = 1, 'apple' = 2)"
+  t << "tags Array(String)"
+end
+```
+
+### Define nested structures
+
+```ruby
+ClickHouse.connection.create_table('visits', engine: 'TinyLog') do |t|
+  t.UInt8 :id
+  t.Nested :json do |n|
+    n.UInt8 :cid
+    n.Date  :created_at
+    n.Date  :updated_at
+  end
+end
+```
+
+### Set table options
+
+```ruby
+ClickHouse.connection.create_table('visits', 
+  order: 'year',
+  ttl: 'date + INTERVAL 1 DAY',
+  sample: 'year',
+  settings: 'index_granularity=8192',
+  primary_key: 'year',
+  engine: 'MergeTree') do |t|
+  t.UInt16  :year
+  t.Date    :date
+end
+```
+
+### Create table with raw SQL
+
+```ruby
+ClickHouse.connection.execute <<~SQL
+  CREATE TABLE visits(int Nullable(Int8), date Nullable(Date)) ENGINE TinyLog
+SQL
+```
+
+## Type casting
+
+By default gem provides all necessary type casting, but you may overwrite or define
+your own logic 
+
+```ruby
+class DateType
+  def cast(value)
+    Date.parse(value)
+  end 
+  
+  def serialize(value)
+    value.strftime('%Y-%m-%d')
+  end
+end
+
+ClickHouse.add_type('Date', DateType.new)
+```
+
+Actually `serialize` function is not used for now, but you may use it manually:
+
+```ruby
+time_type = DateTimeType.new
+string_type = FixedStringType.new 
+
+ClickHouse.connection.insert('table', columns: %i[name time]) do |buffer|
+  buffer << [string_type.serialize('a' * 1000, 20), time.serialize(Time.current, 'Europe/Moscow')]
+end 
+```
+
+If native type supports arguments, define type with `%s` argument:
+
+```ruby
+class DateTimeType
+  def cast(value, time_zone)
+    Time.parse("#{value} #{time_zone}")
+  end 
+end
+
+ClickHouse.add_type('DateTime(%s)', DateTimeType.new)
+```
+
+## Using with a connection pool
+
+```ruby
+require 'connection_pool'
+
+ClickHouse.connection = ConnectionPool.new(size: 2) do
+  ClickHouse::Connection.new(ClickHouse::Config.new(url: 'http://replica.example.com'))
+end
+
+ClickHouse.connection.with do |conn|
+  conn.tables
+end
+```
+
+## Using with Rails
+
+```yml
+# config/click_house.yml
+
+default: &default
+  url: http://localhost:8123
+  timeout: 60
+  open_timeout: 3
+
+development:
+  database: ecliptic_development
+  <<: *default
+
+test:
+  database: ecliptic_test
+  <<: *default
+```
+
+```ruby
+# config/initializers/click_house.rb
+
+ClickHouse.config do |config|
+  config.logger = Rails.logger
+  config.assign(Rails.application.config_for('click_house'))
+end 
+```
+
+```ruby
+# lib/tasks/click_house.rake
+namespace :click_house do
+  task prepare: :environment do
+    @environments = Rails.env.development? ? %w[development test] : [Rails.env]
+  end
+
+  task drop: :prepare do
+    @environments.each do |env|
+      config = ClickHouse.config.clone.assign(Rails.application.config_for('click_house', env: env))
+      connection = ClickHouse::Connection.new(config)
+      connection.drop_database(config.database, if_exists: true)
+    end
+  end
+
+  task create: :prepare do
+    @environments.each do |env|
+      config = ClickHouse.config.clone.assign(Rails.application.config_for('click_house', env: env))
+      connection = ClickHouse::Connection.new(config)
+      connection.create_database(config.database, if_not_exists: true)
+    end
+  end
+end
+```
+
+Prepare the ClickHouse database:
+
+```bash
+rake click_house:drop click_house:create
+```
+
+If your are using SQL Database in Rails, you can manage ClickHouse migrations
+using `ActiveRecord::Migration` mechanism
+
+```ruby
+class CreateAdvertVisits < ActiveRecord::Migration[6.0]
+  def up
+    ClickHouse.connection.create_table('visits', engine: 'MergeTree(date, (account_id, advert_id), 512)') do |t|
+      t.UInt16   :account_id
+      t.UInt16   :user_id
+      t.Date     :date
+      t.DateTime :time, 'UTC'
+      t.UInt32   :ipv4
+      t.UInt64   :ipv6
+      t.UInt32   :device_type_id
+      t.UInt32   :os_family_id
+    end
+  end
+
+  def down    
+    ClickHouse.connection.drop_table('visits')
+  end
+end
+```
+
+if you use `ActiveRecord`, you can use the ORM query builder by using fake models
+
+````ruby
+# FAKE MODEL FOR ClickHouse 
+class Visit < ApplicationRecord
+  scope :with_os, -> { where.not(os_family_id: nil) }
+end
+
+scope = Visit.with_os.select('COUNT(*) as counter').group(:ipv4)
+ClickHouse.connection.select_all(scope.to_sql)
+```` 
+
+## Development
+
+```bash
+make dockerize
+rspec
+rubocop
+```