ch-client 0.0.1

data/lib/clickhouse/connection/logger.rb

@@ -0,0 +1,12 @@
+module Clickhouse
+  class Connection
+    module Logger
+    private
+
+      def log(type, msg)
+        Clickhouse.logger.send(type, msg) if Clickhouse.logger
+      end
+
+    end
+  end
+end

data/lib/clickhouse/connection/query.rb

@@ -0,0 +1,160 @@
+require "clickhouse/connection/query/table"
+require "clickhouse/connection/query/result_set"
+require "clickhouse/connection/query/result_row"
+
+module Clickhouse
+  class Connection
+    module Query
+
+      def execute(query, body = nil)
+        body = post(query, body)
+        body.empty? ? true : body
+      end
+
+      def query(query)
+        query = Utils.extract_format(query)[0]
+        query += " FORMAT JSONCompact"
+        parse_data get(query)
+      end
+
+      def databases
+        query("SHOW DATABASES").flatten
+      end
+
+      def tables
+        query("SHOW TABLES").flatten
+      end
+
+      def create_table(name, &block)
+        execute(Clickhouse::Connection::Query::Table.new(name, &block).to_sql)
+      end
+
+      def describe_table(name)
+        query("DESCRIBE TABLE #{name}").to_a
+      end
+
+      def rename_table(*args)
+        names = (args[0].is_a?(Hash) ? args[0].to_a : [args]).flatten
+        raise Clickhouse::InvalidQueryError, "Odd number of table names" unless (names.size % 2) == 0
+        names = Hash[*names].collect{|(from, to)| "#{from} TO #{to}"}
+        execute("RENAME TABLE #{names.join(", ")}")
+      end
+
+      def drop_table(name)
+        execute("DROP TABLE #{name}")
+      end
+
+      def insert_rows(table, options = {})
+        options[:csv] ||= begin
+          options[:rows] ||= yield([])
+          generate_csv options[:rows], options[:names]
+        end
+        execute("INSERT INTO #{table} FORMAT CSVWithNames", options[:csv])
+      end
+
+      def select_rows(options)
+        query to_select_query(options)
+      end
+
+      def select_row(options)
+        select_rows(options)[0]
+      end
+
+      def select_values(options)
+        select_rows(options).collect{|row| row[0]}
+      end
+
+      def select_value(options)
+        values = select_values(options)
+        values[0] if values
+      end
+
+      def count(options)
+        options = options.merge(:select => "COUNT(*)")
+        select_value(options).to_i
+      end
+
+      def to_select_query(options)
+        to_select_options(options).collect do |(key, value)|
+          next if value.nil? || (value.respond_to?(:empty?) && value.empty?)
+
+          statement = [key.to_s.upcase]
+          statement << "BY" if %W(GROUP ORDER).include?(statement[0])
+          statement << to_segment(key, value)
+          statement.join(" ")
+
+        end.compact.join("\n").force_encoding("UTF-8")
+      end
+
+    private
+
+      def generate_csv(rows, names = nil)
+        hashes = rows[0].is_a?(Hash)
+
+        if hashes
+          names ||= rows[0].keys
+        end
+
+        CSV.generate do |csv|
+          csv << names if names
+          rows.each do |row|
+            csv << (hashes ? row.values_at(*names) : row)
+          end
+        end
+      end
+
+      def inspect_value(value)
+        value.nil? ? "NULL" : value.inspect.gsub(/(^"|"$)/, "'").gsub("\\\"", "\"")
+      end
+
+      def to_select_options(options)
+        keys = [:select, :from, :where, :group, :having, :order, :limit, :offset]
+
+        options = Hash[keys.zip(options.values_at(*keys))]
+        options[:select] ||= "*"
+        options[:limit] ||= 0 if options[:offset]
+        options[:limit] = options.values_at(:offset, :limit).compact.join(", ") if options[:limit]
+        options.delete(:offset)
+
+        options
+      end
+
+      def to_segment(type, value)
+        case type
+        when :select
+          [value].flatten.join(", ")
+        when :where, :having
+          value.is_a?(Hash) ? to_condition_statements(value) : value
+        else
+          value
+        end
+      end
+
+      def to_condition_statements(value)
+        value.collect do |attr, val|
+          if val == :empty
+            "empty(#{attr})"
+          elsif val.is_a?(Range)
+            [
+              "#{attr} >= #{inspect_value(val.first)}",
+              "#{attr} <= #{inspect_value(val.last)}"
+            ]
+          elsif val.is_a?(Array)
+            "#{attr} IN (#{val.collect{|x| inspect_value(x)}.join(", ")})"
+          elsif val.to_s.match(/^`.*`$/)
+            "#{attr} #{val.gsub(/(^`|`$)/, "")}"
+          else
+            "#{attr} = #{inspect_value(val)}"
+          end
+        end.flatten.join(" AND ")
+      end
+
+      def parse_data(data)
+        names = data["meta"].collect{|column| column["name"]}
+        types = data["meta"].collect{|column| column["type"]}
+        ResultSet.new data["data"], names, types
+      end
+
+    end
+  end
+end

data/lib/clickhouse/connection/query/result_row.rb

@@ -0,0 +1,36 @@
+module Clickhouse
+  class Connection
+    module Query
+      class ResultRow < Array
+
+        def initialize(values = [], keys = nil)
+          super values
+          @keys = normalize_keys(keys)
+        end
+
+        def to_hash(symbolize = false)
+          @hash ||= begin
+            keys = symbolize ? @keys.collect(&:to_sym) : @keys
+            Hash[keys.zip(self)]
+          end
+        end
+
+      private
+
+        def normalize_keys(keys)
+          if keys
+            keys.collect do |key|
+              key.match(/^any\(([^\)]+)\)$/)
+              $1 || key
+            end
+          else
+            (0..(size - 1)).collect do |index|
+              "column#{index}"
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/clickhouse/connection/query/result_set.rb

@@ -0,0 +1,103 @@
+module Clickhouse
+  class Connection
+    module Query
+      class ResultSet
+        include Enumerable
+        extend Forwardable
+
+        def_delegators :@rows, :size, :empty?
+        def_delegators :to_a, :first, :last, :flatten
+
+        attr_reader :names, :types
+
+        def initialize(rows = [], names = nil, types = nil)
+          @rows = rows
+          @names = names
+          @types = types
+        end
+
+        def each
+          (0..(size - 1)).collect do |index|
+            yield self[index]
+            self[index]
+          end
+        end
+
+        def [](index)
+          row = @rows[index]
+          row = @rows[index] = parse_row(row) if row.class == Array
+          row
+        end
+
+        def present?
+          !empty?
+        end
+
+        def to_hashes(symbolize = false)
+          collect{|row| row.to_hash(symbolize)}
+        end
+
+      private
+
+        def parse_row(array)
+          values = array.each_with_index.to_a.collect do |value, i|
+            parse_value(@types[i], value) if @types
+          end
+          ResultRow.new values, @names
+        end
+
+        def parse_value(type, value)
+          if value
+            case type
+            when "UInt8", "UInt16", "UInt32", "UInt64", "Int8", "Int16", "Int32", "Int64"
+              parse_int_value value
+            when "Float32", "Float64"
+              parse_float_value value
+            when "String", "Enum8", "Enum16"
+              parse_string_value value
+            when /FixedString\(\d+\)/
+              parse_fixed_string_value value
+            when "Date"
+              parse_date_value value
+            when "DateTime"
+              parse_date_time_value value
+            when /Array\(/
+              parse_array_value value
+            else
+              raise NotImplementedError, "Cannot parse value of type #{type.inspect}"
+            end
+          end
+        end
+
+        def parse_int_value(value)
+          value.to_i
+        end
+
+        def parse_float_value(value)
+          value.to_f
+        end
+
+        def parse_string_value(value)
+          value.force_encoding("UTF-8")
+        end
+
+        def parse_fixed_string_value(value)
+          value.delete("\000").force_encoding("UTF-8")
+        end
+
+        def parse_date_value(value)
+          Date.parse(value)
+        end
+
+        def parse_date_time_value(value)
+          Time.parse(value)
+        end
+
+        def parse_array_value(value)
+          value
+        end
+
+      end
+    end
+  end
+end

data/lib/clickhouse/connection/query/table.rb

@@ -0,0 +1,50 @@
+module Clickhouse
+  class Connection
+    module Query
+      class Table
+
+        def initialize(name)
+          @name = name
+          @columns = []
+          yield self
+        end
+
+        def engine(value)
+          @engine = value
+        end
+
+        def to_sql
+          raise Clickhouse::InvalidQueryError, "Missing table engine" unless @engine
+          length = @columns.collect{|x| x[0].to_s.size}.max
+
+          sql = []
+          sql << "CREATE TABLE #{@name} ("
+
+          @columns.each_with_index do |(name, type), index|
+            sql << "  #{name.ljust(length, " ")} #{type}#{"," unless index == @columns.size - 1}"
+          end
+
+          sql << ")"
+          sql << "ENGINE = #{@engine}"
+
+          sql.join("\n")
+        end
+
+      private
+
+        def method_missing(name, *args)
+          type = name.to_s
+                  .gsub(/(^.|_\w)/) {
+                    $1.upcase
+                  }
+                  .gsub("Uint", "UInt")
+                  .delete("_")
+
+          type << "(#{args[1]})" if args[1]
+          @columns << [args[0].to_s, type]
+        end
+
+      end
+    end
+  end
+end

data/lib/clickhouse/error.rb

@@ -0,0 +1,18 @@
+module Clickhouse
+
+  class Error < StandardError
+  end
+
+  class ConnectionError < Error
+  end
+
+  class InvalidConnectionError < ConnectionError
+  end
+
+  class QueryError < Error
+  end
+
+  class InvalidQueryError < QueryError
+  end
+
+end

data/lib/clickhouse/utils.rb

@@ -0,0 +1,23 @@
+module Clickhouse
+  module Utils
+    extend self
+
+    def normalize_url(url)
+      if url.match(/^\w+:\/\//)
+        url
+      else
+        "#{Clickhouse::Connection::DEFAULT_CONFIG[:scheme]}://#{url}"
+      end
+    end
+
+    def extract_format(query)
+      format = nil
+      query = query.gsub(/ FORMAT (\w+)/i) do
+        format = $1
+        ""
+      end
+      [query.strip, format]
+    end
+
+  end
+end

data/lib/clickhouse/version.rb

@@ -0,0 +1,7 @@
+module Clickhouse
+  MAJOR = 0
+  MINOR = 0
+  TINY  = 1
+
+  VERSION = [MAJOR, MINOR, TINY].join(".")
+end

data/script/console

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require "logger"
+require "bundler"
+Bundler.require :default, :development
+
+def connect!(config = {})
+  Clickhouse.logger = Logger.new(STDOUT)
+  Clickhouse.establish_connection(config)
+end
+
+def conn(config = {})
+  connect!(config) unless Clickhouse.connection
+  Clickhouse.connection
+end
+
+def events
+  "events"
+end
+
+def create_table
+  conn.create_table(events) do |t|
+    t.fixed_string :id, 16
+    t.uint16       :year
+    t.date         :date
+    t.date_time    :time
+    t.string       :event
+    t.uint32       :user_id
+    t.float32      :revenue
+    t.engine       "MergeTree(date, (year, date), 8192)"
+  end
+end
+
+def insert_rows
+  conn.insert_rows(events, :names => %w(id year date time event user_id revenue)) do |rows|
+    rows << [
+      "d91d1c90",
+      2016,
+      "2016-10-17",
+      "2016-10-17 23:14:28",
+      "click",
+      1982,
+      0.18
+    ]
+    rows << [
+      "d91d2294",
+      2016,
+      "2016-10-17",
+      "2016-10-17 23:14:41",
+      "click",
+      1947,
+      0.203
+    ]
+  end
+end
+
+puts "Loading Clickhouse development environment (#{Clickhouse::VERSION})"
+Pry.start