ch-client 0.0.1

data/lib/clickhouse/cli/server/assets/js/sql.js

@@ -0,0 +1,232 @@
+CodeMirror.defineMode("sql", function(config, parserConfig) {
+  "use strict";
+
+  var client         = parserConfig.client || {},
+      atoms          = parserConfig.atoms || {"false": true, "true": true, "null": true},
+      builtin        = parserConfig.builtin || {},
+      keywords       = parserConfig.keywords,
+      operatorChars  = /^[*+\-%<>!=&|~^]/,
+      hooks          = parserConfig.hooks || {};
+
+  function tokenBase(stream, state) {
+    var ch = stream.next();
+
+    if (hooks[ch]) {
+      var result = hooks[ch](stream, state);
+      if (result !== false) return result;
+    }
+
+    if ((ch == "0" && stream.match(/^[xX][0-9a-fA-F]+/))
+	|| (ch == "x" || ch == "X") && stream.match(/^'[0-9a-fA-F]+'/)) {
+      // hex
+      return "number";
+    } else if (((ch == "b" || ch == "B") && stream.match(/^'[01]+'/))
+	       || (ch == "0" && stream.match(/^b[01]+/))) {
+      // bitstring
+      return "number";
+    } else if (ch.charCodeAt(0) > 47 && ch.charCodeAt(0) < 58) {
+      // numbers
+      stream.match(/^[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?/);
+      return "number";
+    } else if (ch == "?" && (stream.eatSpace() || stream.eol() || stream.eat(";"))) {
+      // placeholders
+      return "variable-3";
+    } else if (ch == '"' || ch == "'") {
+      state.tokenize = tokenLiteral(ch);
+      return state.tokenize(stream, state);
+    } else if (/^[\(\),\.;\[\]]/.test(ch)) {
+      return null;
+    } else if (ch == "#" || (ch == "-" && stream.eat("-") && stream.eat(" "))) {
+      stream.skipToEnd();
+      return "comment";
+    } else if (ch == "/" && stream.eat("*")) {
+      state.tokenize = tokenComment;
+      return state.tokenize(stream, state);
+    } else if (operatorChars.test(ch)) {
+      stream.eatWhile(operatorChars);
+      return false;
+    } else {
+      stream.eatWhile(/^[_\w\d]/);
+      var word = stream.current().toLowerCase();
+      if (atoms.hasOwnProperty(word)) return "atom";
+      if (builtin.hasOwnProperty(word)) return "builtin";
+      if (keywords.hasOwnProperty(word)) return "keyword";
+      if (client.hasOwnProperty(word)) return "string-2";
+      return "variable";
+    }
+  }
+
+  function tokenLiteral(quote) {
+    return function(stream, state) {
+      var escaped = false, ch;
+      while ((ch = stream.next()) != null) {
+	if (ch == quote && !escaped) {
+	  state.tokenize = tokenBase;
+	  break;
+	}
+	escaped = !escaped && ch == "\\";
+      }
+      return "string";
+    };
+  }
+  function tokenComment(stream, state) {
+    while (true) {
+      if (stream.skipTo("*")) {
+	stream.next();
+	if (stream.eat("/")) {
+	  state.tokenize = tokenBase;
+	  break;
+	}
+      } else {
+	stream.skipToEnd();
+	break;
+      }
+    }
+    return "comment";
+  }
+
+  function pushContext(stream, state, type) {
+    state.context = {
+      prev: state.context,
+      indent: stream.indentation(),
+      col: stream.column(),
+      type: type
+    };
+  }
+
+  function popContext(state) {
+    state.indent = state.context.indent;
+    state.context = state.context.prev;
+  }
+
+  return {
+    startState: function() {
+      return {tokenize: tokenBase, context: null};
+    },
+
+    token: function(stream, state) {
+      if (stream.sol()) {
+	if (state.context && state.context.align == null)
+	  state.context.align = false;
+      }
+      if (stream.eatSpace()) return null;
+
+      var style = state.tokenize(stream, state);
+      if (style == "comment") return style;
+
+      if (state.context && state.context.align == null)
+	state.context.align = true;
+
+      var tok = stream.current();
+      if (tok == "(")
+        pushContext(stream, state, ")");
+      else if (tok == "[")
+        pushContext(stream, state, "]");
+      else if (state.context && state.context.type == tok)
+        popContext(state);
+      return style;
+    },
+
+    indent: function(state, textAfter) {
+      var cx = state.context;
+      if (!cx) return CodeMirror.Pass;
+      if (cx.align) return cx.col + (textAfter.charAt(0) == cx.type ? 0 : 1);
+      else return cx.indent + config.indentUnit;
+    }
+  };
+});
+
+(function() {
+  "use strict";
+
+  function hookIdentifier(stream) {
+    var escaped = false, ch;
+
+    while ((ch = stream.next()) != null) {
+      if (ch == "`" && !escaped) return "variable-2";
+      escaped = !escaped && ch == "`";
+    }
+    return false;
+  }
+
+  // variable token
+  function hookVar(stream) {
+    // variables
+    // @@ and prefix
+    if (stream.eat("@")) {
+      stream.match(/^session\./);
+      stream.match(/^local\./);
+      stream.match(/^global\./);
+    }
+
+    if (stream.eat("'")) {
+      stream.match(/^.*'/);
+      return "variable-2";
+    } else if (stream.eat('"')) {
+      stream.match(/^.*"/);
+      return "variable-2";
+    } else if (stream.eat("`")) {
+      stream.match(/^.*`/);
+      return "variable-2";
+    } else if (stream.match(/^[0-9a-zA-Z$\.\_]+/)) {
+      return "variable-2";
+    }
+    return false;
+  };
+
+  // short client keyword token
+  function hookClient(stream) {
+    // \g, etc
+    return stream.match(/^[a-zA-Z]\b/) ? "variable-2" : false;
+  }
+
+  var sqlKeywords = "alter and as asc between by count create delete desc distinct drop from having in insert into is join like not on or order select set table union update values where ";
+
+  function set(str) {
+    var obj = {}, words = str.split(" ");
+    for (var i = 0; i < words.length; ++i) obj[words[i]] = true;
+    return obj;
+  }
+
+  CodeMirror.defineMIME("text/x-sql", {
+    name: "sql",
+    keywords: set(sqlKeywords + "begin"),
+    builtin: set("bool boolean bit blob enum long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text bigint int int1 int2 int3 int4 int8 integer float float4 float8 double char varbinary varchar varcharacter precision real date datetime year unsigned signed decimal numeric"),
+    atoms: set("false true null unknown")
+  });
+
+  CodeMirror.defineMIME("text/x-mysql", {
+    name: "sql",
+    client: set("charset clear connect edit ego exit go help nopager notee nowarning pager print prompt quit rehash source status system tee"),
+    keywords: set(sqlKeywords + "accessible action add after algorithm all analyze asensitive at authors auto_increment autocommit avg avg_row_length before binary binlog both btree cache call cascade cascaded case catalog_name chain change changed character check checkpoint checksum class_origin client_statistics close coalesce code collate collation collations column columns comment commit committed completion concurrent condition connection consistent constraint contains continue contributors convert cross current_date current_time current_timestamp current_user cursor data database databases day_hour day_microsecond day_minute day_second deallocate dec declare default delay_key_write delayed delimiter des_key_file describe deterministic dev_pop dev_samp deviance directory disable discard distinctrow div dual dumpfile each elseif enable enclosed end ends engine engines enum errors escape escaped even event events every execute exists exit explain extended fast fetch field fields first flush for force foreign found_rows full fulltext function general global grant grants group groupby_concat handler hash help high_priority hosts hour_microsecond hour_minute hour_second if ignore ignore_server_ids import index index_statistics infile inner innodb inout insensitive insert_method interval invoker isolation iterate key keys kill language last leading leave left level limit linear lines list load local localtime localtimestamp lock logs low_priority master master_heartbeat_period master_ssl_verify_server_cert masters match max max_rows maxvalue message_text middleint migrate min min_rows minute_microsecond minute_second mod mode modifies modify mutex mysql_errno natural next no no_write_to_binlog offline offset one online open optimize option optionally out outer outfile pack_keys parser partition partitions password phase plugin plugins prepare preserve prev primary privileges procedure processlist profile profiles purge query quick range read read_write reads real rebuild recover references regexp relaylog release remove rename reorganize repair repeatable replace require resignal restrict resume return returns revoke right rlike rollback rollup row row_format rtree savepoint schedule schema schema_name schemas second_microsecond security sensitive separator serializable server session share show signal slave slow smallint snapshot spatial specific sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_no_cache sql_small_result sqlexception sqlstate sqlwarning ssl start starting starts status std stddev stddev_pop stddev_samp storage straight_join subclass_origin sum suspend table_name table_statistics tables tablespace temporary terminated to trailing transaction trigger triggers truncate uncommitted undo unique unlock upgrade usage use use_frm user user_resources user_statistics using utc_date utc_time utc_timestamp value variables varying view views warnings when while with work write xa xor year_month zerofill begin do then else loop repeat"),
+    builtin: set("bool boolean bit blob enum long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text bigint int int1 int2 int3 int4 int8 integer float float4 float8 double char varbinary varchar varcharacter precision date datetime year unsigned signed decimal numeric"),
+    atoms: set("false true null unknown"),
+    hooks: {
+      "@":   hookVar,
+      "`":   hookIdentifier,
+      "\\":  hookClient
+    }
+  });
+
+  CodeMirror.defineMIME("text/x-mariadb", {
+    name: "sql",
+    client: set("charset clear connect edit ego exit go help nopager notee nowarning pager print prompt quit rehash source status system tee"),
+    keywords: set(sqlKeywords + "accessible action add after algorithm all always analyze asensitive at authors auto_increment autocommit avg avg_row_length before binary binlog both btree cache call cascade cascaded case catalog_name chain change changed character check checkpoint checksum class_origin client_statistics close coalesce code collate collation collations column columns comment commit committed completion concurrent condition connection consistent constraint contains continue contributors convert cross current_date current_time current_timestamp current_user cursor data database databases day_hour day_microsecond day_minute day_second deallocate dec declare default delay_key_write delayed delimiter des_key_file describe deterministic dev_pop dev_samp deviance directory disable discard distinctrow div dual dumpfile each elseif enable enclosed end ends engine engines enum errors escape escaped even event events every execute exists exit explain extended fast fetch field fields first flush for force foreign found_rows full fulltext function general generated global grant grants group groupby_concat handler hash help high_priority hosts hour_microsecond hour_minute hour_second if ignore ignore_server_ids import index index_statistics infile inner innodb inout insensitive insert_method interval invoker isolation iterate key keys kill language last leading leave left level limit linear lines list load local localtime localtimestamp lock logs low_priority master master_heartbeat_period master_ssl_verify_server_cert masters match max max_rows maxvalue message_text middleint migrate min min_rows minute_microsecond minute_second mod mode modifies modify mutex mysql_errno natural next no no_write_to_binlog offline offset one online open optimize option optionally out outer outfile pack_keys parser partition partitions password persistent phase plugin plugins prepare preserve prev primary privileges procedure processlist profile profiles purge query quick range read read_write reads real rebuild recover references regexp relaylog release remove rename reorganize repair repeatable replace require resignal restrict resume return returns revoke right rlike rollback rollup row row_format rtree savepoint schedule schema schema_name schemas second_microsecond security sensitive separator serializable server session share show signal slave slow smallint snapshot spatial specific sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_no_cache sql_small_result sqlexception sqlstate sqlwarning ssl start starting starts status std stddev stddev_pop stddev_samp storage straight_join subclass_origin sum suspend table_name table_statistics tables tablespace temporary terminated to trailing transaction trigger triggers truncate uncommitted undo unique unlock upgrade usage use use_frm user user_resources user_statistics using utc_date utc_time utc_timestamp value variables varying view views virtual warnings when while with work write xa xor year_month zerofill begin do then else loop repeat"),
+    builtin: set("bool boolean bit blob enum long longblob longtext medium mediumblob mediumint mediumtext time timestamp tinyblob tinyint tinytext text bigint int int1 int2 int3 int4 int8 integer float float4 float8 double char varbinary varchar varcharacter precision date datetime year unsigned signed decimal numeric"),
+    atoms: set("false true null unknown"),
+    hooks: {
+      "@":   hookVar,
+      "`":   hookIdentifier,
+      "\\":  hookClient
+    }
+  });
+
+  // this is based on Peter Raganitsch's 'plsql' mode
+  CodeMirror.defineMIME("text/x-plsql", {
+    name:       "sql",
+    client:     set("appinfo arraysize autocommit autoprint autorecovery autotrace blockterminator break btitle cmdsep colsep compatibility compute concat copycommit copytypecheck define describe echo editfile embedded escape exec execute feedback flagger flush heading headsep instance linesize lno loboffset logsource long longchunksize markup native newpage numformat numwidth pagesize pause pno recsep recsepchar release repfooter repheader serveroutput shiftinout show showmode size spool sqlblanklines sqlcase sqlcode sqlcontinue sqlnumber sqlpluscompatibility sqlprefix sqlprompt sqlterminator suffix tab term termout time timing trimout trimspool ttitle underline verify version wrap"),
+    keywords:   set("abort accept access add all alter and any array arraylen as asc assert assign at attributes audit authorization avg base_table begin between binary_integer body boolean by case cast char char_base check close cluster clusters colauth column comment commit compress connect connected constant constraint crash create current currval cursor data_base database date dba deallocate debugoff debugon decimal declare default definition delay delete desc digits dispose distinct do drop else elsif enable end entry escape exception exception_init exchange exclusive exists exit external fast fetch file for force form from function generic goto grant group having identified if immediate in increment index indexes indicator initial initrans insert interface intersect into is key level library like limited local lock log logging long loop master maxextents maxtrans member minextents minus mislabel mode modify multiset new next no noaudit nocompress nologging noparallel not nowait number_base object of off offline on online only open option or order out package parallel partition pctfree pctincrease pctused pls_integer positive positiven pragma primary prior private privileges procedure public raise range raw read rebuild record ref references refresh release rename replace resource restrict return returning reverse revoke rollback row rowid rowlabel rownum rows run savepoint schema segment select separate session set share snapshot some space split sql start statement storage subtype successful synonym tabauth table tables tablespace task terminate then to trigger truncate type union unique unlimited unrecoverable unusable update use using validate value values variable view views when whenever where while with work"),
+    functions:  set("abs acos add_months ascii asin atan atan2 average bfilename ceil chartorowid chr concat convert cos cosh count decode deref dual dump dup_val_on_index empty error exp false floor found glb greatest hextoraw initcap instr instrb isopen last_day least lenght lenghtb ln lower lpad ltrim lub make_ref max min mod months_between new_time next_day nextval nls_charset_decl_len nls_charset_id nls_charset_name nls_initcap nls_lower nls_sort nls_upper nlssort no_data_found notfound null nvl others power rawtohex reftohex round rowcount rowidtochar rpad rtrim sign sin sinh soundex sqlcode sqlerrm sqrt stddev substr substrb sum sysdate tan tanh to_char to_date to_label to_multi_byte to_number to_single_byte translate true trunc uid upper user userenv variance vsize"),
+    builtin:    set("bfile blob character clob dec float int integer mlslabel natural naturaln nchar nclob number numeric nvarchar2 real rowtype signtype smallint string varchar varchar2")
+  });
+}());

data/lib/clickhouse/cli/server/views/index.erb

@@ -0,0 +1,46 @@
+<html>
+  <head>
+    <title>Clickhouse</title>
+    <link rel="stylesheet" href="/css/normalize.css">
+    <link rel="stylesheet" href="/css/skeleton.css">
+    <link rel="stylesheet" href="/css/codemirror.css">
+    <link rel="stylesheet" href="/css/datatables.css">
+    <link rel="stylesheet" href="/css/clickhouse.css">
+    <script charset="utf-8" src="/js/jquery.js"></script>
+    <script charset="utf-8" src="/js/codemirror.js"></script>
+    <script charset="utf-8" src="/js/datatables.js"></script>
+    <script charset="utf-8" src="/js/clickhouse.js"></script>
+    <script charset="utf-8" src="/js/sql.js"></script>
+  </head>
+  <body>
+    <h6 class="header">
+      Clickhouse Client
+      <small>&nbsp;/&nbsp; <span id="urls"></span></small>
+    </h6>
+    <div class="container">
+      <div class="row">
+        <!--
+        <div class="three columns">
+          Recent queries
+        </div>
+        -->
+        <div class="twelve columns">
+          <!-- <h6>Compose query</h6> -->
+          <form>
+            <textarea id="sql" name="sql"></textarea>
+            <input class="button" type="submit" value="Run query">
+            <a href="#" class="button download">Save as TSV</a>
+            <span id="stats"></span>
+          </form>
+        </div>
+      </div>
+    </div>
+    <script type="text/javascript">
+      Clickhouse.init(<%=
+        Readline::HISTORY.to_a.inspect
+      %>, <%=
+        Clickhouse.connection.pond.available.collect(&:url).inspect
+      %>);
+    </script>
+  </body>
+</html>

data/lib/clickhouse/cluster.rb

@@ -0,0 +1,43 @@
+module Clickhouse
+  class Cluster < BasicObject
+
+    attr_reader :pond
+
+    def initialize(config)
+      config = config.dup
+      urls = config.delete(:urls) || config.delete("urls")
+      urls.collect!{|url| ::Clickhouse::Utils.normalize_url(url)}
+
+      @pond = ::Pond.new :maximum_size => urls.size, :timeout => 5.0
+      block = ::Proc.new do
+        url = (urls - pond.available.collect(&:url)).first || urls.sample
+        ::Clickhouse::Connection.new(config.merge(:url => url))
+      end
+
+      pond.instance_variable_set :@block, block
+      pond.maximum_size.times do
+        pond.available << block.call
+      end
+
+      pond.detach_if = ::Proc.new do |connection|
+        begin
+          connection.ping!
+          false
+        rescue
+          true
+        end
+      end
+    end
+
+  private
+
+    def method_missing(*args, &block)
+      pond.checkout do |connection|
+        connection.send(*args, &block)
+      end
+    rescue ::Clickhouse::ConnectionError
+      retry if pond.available.any?
+    end
+
+  end
+end

data/lib/clickhouse/connection.rb

@@ -0,0 +1,42 @@
+require "clickhouse/connection/client"
+require "clickhouse/connection/logger"
+require "clickhouse/connection/query"
+
+module Clickhouse
+  class Connection
+
+    DEFAULT_CONFIG = {
+      :scheme => "http",
+      :host => "localhost",
+      :port => 8123
+    }
+
+    include Client
+    include Logger
+    include Query
+
+    def initialize(config = {})
+      @config = normalize_config(config)
+    end
+
+  private
+
+    def normalize_config(config)
+      config = config.inject({}) do |hash, (key, value)|
+        hash[key.to_sym] = value
+        hash
+      end
+
+      if config[:url]
+        uri = URI Clickhouse::Utils.normalize_url(config[:url])
+        config[:scheme] = uri.scheme
+        config[:host] = uri.host
+        config[:port] = uri.port
+        config.delete(:url)
+      end
+
+      DEFAULT_CONFIG.merge(config.reject{|_k, v| v.nil?})
+    end
+
+  end
+end

data/lib/clickhouse/connection/client.rb

@@ -0,0 +1,135 @@
+module Clickhouse
+  class Connection
+    module Client
+      include ActiveSupport::NumberHelper
+
+      def connect!
+        ping! unless connected?
+      end
+
+      def ping!
+        ensure_authentication
+        status = client.get("/").status
+        if status != 200
+          raise ConnectionError, "Unexpected response status: #{status}"
+        end
+        true
+      rescue Faraday::Error => e
+        raise ConnectionError, e.message
+      end
+
+      def connected?
+        instance_variables.include?(:@client) && !!@client
+      end
+
+      def get(query)
+        request(:get, query)
+      end
+
+      def post(query, body = nil)
+        request(:post, query, body)
+      end
+
+      def url
+        "#{@config[:scheme]}://#{@config[:host]}:#{@config[:port]}"
+      end
+
+    private
+
+      def client
+        @client ||= Faraday.new(:url => url)
+      end
+
+      def ensure_authentication
+        username, password = @config.values_at(:username, :password)
+        client.basic_auth(username || "default", password) if username || password
+      end
+
+      def path(query)
+        params = @config.select{|k, _v| k == :database}
+        params[:query] = query
+        params[:output_format_write_statistics] = 1
+        query_string = params.collect{|k, v| "#{k}=#{CGI.escape(v.to_s)}"}.join("&")
+
+        "/?#{query_string}"
+      end
+
+      def request(method, query, body = nil)
+        connect!
+        query = query.strip
+        start = Time.now
+
+        response = client.send(method, path(query), body)
+        status = response.status
+        duration = Time.now - start
+        query, format = Utils.extract_format(query)
+        response = parse_body(format, response.body)
+        stats = parse_stats(response)
+
+        write_log duration, query, stats
+        raise QueryError, "Got status #{status} (expected 200): #{response}" unless status == 200
+        response
+
+      rescue Faraday::Error => e
+        raise ConnectionError, e.message
+      end
+
+      def parse_body(format, body)
+        case format
+        when "JSON", "JSONCompact"
+          body = JSON.parse(body) if body.strip[0] == "{"
+        end
+        body
+      end
+
+      def parse_stats(response)
+        return {} unless response.is_a?(Hash)
+
+        options = {:locale => :en, :precision => 2, :significant => false}
+        stats = response["statistics"].merge("rows" => response["rows"])
+        factor = 1 / stats["elapsed"]
+
+        stats["elapsed"] = number_to_human_duration(stats["elapsed"])
+        stats["rows_per_second"] = number_to_human(stats["rows_read"] * factor, options).downcase
+        stats["data_per_second"] = number_to_human_size(stats["bytes_read"] * factor, options)
+        stats["rows_read"] = number_to_human(stats["rows_read"], options).downcase
+        stats["data_read"] = number_to_human_size(stats["bytes_read"], options)
+
+        stats
+      end
+
+      def write_log(duration, query, stats)
+        duration = number_to_human_duration(duration)
+
+        rows,
+        elapsed,
+        rows_read,
+        data_read,
+        rows_per_second,
+        data_per_second = stats.values_at(*%w(
+          rows
+          elapsed
+          rows_read
+          data_read
+          rows_per_second
+          data_per_second
+        ))
+
+        line1 = "\n \e[1mSQL (#{duration})\e[0m  #{query};"
+        line2 = "\n  \e[1m#{rows} #{"row".pluralize(rows)} in set. Elapsed: #{elapsed}. Processed: #{rows_read} rows, #{data_read} (#{rows_per_second} rows/s, #{data_per_second}/s)\e[0m" if rows
+
+        log :debug, "#{line1}#{line2} "
+      end
+
+      def number_to_human_duration(number)
+        if (amount = number * 1000.0) < 1000
+          round = (amount < 1) ? 3 : 1
+          "#{amount.round(round)}ms"
+        else
+          "#{number.round(1)}s"
+        end
+      end
+
+    end
+  end
+end