tair 0.1.0.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 02165903d5bd1c58ab4b7d9f005c8d60d2010f2a
+  data.tar.gz: cdcae2a4411765d813550b97e868e10af5bc0291
+SHA512:
+  metadata.gz: 6bf80ccb4622346a4aafb25e439799a90792356a4091304124f270853614c1a3476473cb3be546ae707c634c532a71deb2acce9e7167c05c20aa837e5b574394
+  data.tar.gz: b6f4d76b33be133a1834db4402df6c9d22326a4a99f0caee953eb9135225d7c3b17ddc1643e21cdee4f5bd0d026151cc5a23427114b7d5d27d24cac05573f82c

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+log/*.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://ruby.taobao.org'
+
+# Specify your gem's dependencies in ruby-tair.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 如彼
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rake/extensiontask"
+
+Rake::ExtensionTask.new "tair" do |ext|
+  ext.name = 'tair_murmurhash'
+end

data/ext/tair/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf-rice'
+
+create_makefile "tair_murmurhash"

data/ext/tair/tair_murmurhash.cpp

@@ -0,0 +1,72 @@
+#include "rice/Module.hpp"
+#include "rice/Class.hpp"
+#include "rice/String.hpp"
+
+using namespace Rice;
+using namespace std;
+
+#define MURMURHASH_M 0x5bd1e995
+
+uint64_t digest(string str, uint32_t seed)
+{
+  const char *key = str.c_str();
+  int len         = str.length();
+  uint32_t h      = seed ^ len;
+  int index       = 0;
+
+  while (len >= 4) {
+    uint32_t k = (key[index] & 0xff) | ((key[index + 1] << 8) & 0xff00)
+      | ((key[index + 2] << 16) & 0xff0000)
+      | (key[index + 3] << 24);
+
+    k *= MURMURHASH_M;
+    k ^= (k >> 24);
+    k *= MURMURHASH_M;
+    h *= MURMURHASH_M;
+    h ^= k;
+    index += 4;
+    len -= 4;
+  }
+
+  switch (len) {
+    case 3:
+      h ^= (key[index + 2] << 16);
+
+    case 2:
+      h ^= (key[index + 1] << 8);
+
+    case 1:
+      h ^= key[index];
+      h *= MURMURHASH_M;
+  }
+
+  h ^= (h >> 13);
+  h *= MURMURHASH_M;
+  h ^= (h >> 15);
+  return h;
+}
+
+
+Object tmh_initialize(Object self, String str, uint32_t seed)
+{
+  self.iv_set("@string", str);
+  self.iv_set("@seed", seed);
+}
+
+Object tmh_digest(Object self)
+{
+  string str    = self.iv_get("@string").to_s().str();
+  uint32_t seed = from_ruby<uint32_t>(self.iv_get("@seed"));
+  uint32_t hash = digest(str, seed);
+  return to_ruby<uint32_t>(hash);
+}
+
+extern "C"
+void Init_tair_murmurhash()
+{
+  Module rb_mTair = define_module("Tair");
+  Class rb_cTairMurmurhash = 
+    define_class_under(rb_mTair, "TairMurmurhash")
+    .define_method( "initialize", &tmh_initialize )
+    .define_method( "digest",     &tmh_digest );
+}

data/lib/tair.rb

@@ -0,0 +1,8 @@
+require 'bindata'
+
+require "tair/version"
+require "tair/error"
+require "tair/protocol"
+require "tair/client"
+require 'tair/key'
+require "tair/log"

data/lib/tair/client.rb

@@ -0,0 +1,107 @@
+require 'tair/cluster'
+require 'tair/key'
+require 'tair/request'
+require 'tair/response'
+require 'tair/connection'
+require 'tair/protocol/murmurhash'
+
+Dir.glob(File.expand_path '../operation/*.rb', __FILE__) do |f|
+  require f
+end
+
+module Tair
+
+  class Client
+
+    attr_accessor :namespace
+
+    include Operation
+    include Log
+
+    def self.create(host, port, opts={}, &block)
+      new(host, port, opts).tap do |client|
+        block && block.call(client)
+      end
+    end
+
+
+    attr_reader :host, :port, :group
+    attr_writer :data_servers
+
+    def initialize(host, port, group: nil, namespace: 0xab)
+      @host, @port, @group, @namespace = host, port, group, namespace
+    end
+
+
+    # 配置有变化时，需要更新到客户端
+    def watch_cluster_config(interval=120)
+      Thread.new { watch_cluster_config!(interval) }
+    end
+
+
+    def watch_cluster_config!(interval=120)
+      while true
+        sleep interval
+        @cluster = fetch_data_servers(group)
+      end
+    end
+
+
+    WRITE_OPERATIONS = [:put, :incr, :set_count, :delete]
+
+    def operate(op, request, key: nil, &block)
+      request.namespace = namespace
+      will_write_data = WRITE_OPERATIONS.include? op
+      conn = data_conn(key, will_write_data: will_write_data)
+      log_time "#{op} #{key.inspect} on #{conn.host}" do
+        conn.operate(request, &block)
+      end
+    end
+
+
+    def init_cluster(retries: 5)
+      count = retries
+      begin
+        cluster
+      rescue => e
+        error { "init cluster error: #{e}" }
+        count -= 1
+        if count >= 0
+          retry
+        else
+          raise
+        end
+      end
+    end
+
+
+    private
+
+      # 目前读写对应的节点都是一样的，以后
+      # 支持 down_servers 配置时，就可能不
+      # 同
+      def data_conn(key, will_write_data: false)
+        node = cluster.data_node_for(key, will_write_data: will_write_data)
+        Connection.new(*node.split(':')).tap do |cn|
+          cn.logger = logger
+        end
+      end
+
+      alias_method :conn, :data_conn
+
+
+      def cluster
+        @cluster ||= fetch_data_servers(group)
+      end
+
+
+      def config_conn
+        opts = {}
+        @config_conn ||= Connection.new(host, port, opts)
+      end
+
+
+
+  end
+  
+end

data/lib/tair/cluster.rb

@@ -0,0 +1,51 @@
+module Tair
+
+  # 一个 cluster 是一个 tair 节点组合
+  # cluster 由 bucket 构成
+  # bucket  由 data_node 构成
+  # node -> bucket -> cluster
+  class Cluster
+
+    attr_reader :nodes_per_bucket, :copying_buckets_count
+
+
+    # 根据指定的值，找到对应的数据节点
+    def data_node_for key, will_write_data: false
+      tair_key = Tair::TairObject.from(key).to_binary_s[4..-1]
+      digest   = Tair::Protocol::Murmurhash.digest tair_key, 97
+      idx      = digest % nodes_per_bucket
+      primary_nodes[idx]
+    end
+
+
+    # 更新集群信息
+    #
+    # Params:
+    #   - nodes Array 所有节点数组
+    #   - nodes_per_bucket Integer 多少个节点构成一个 bucket
+    #   - bucket_for_copy  Integer 有多少个 bucket 用来作备库
+    def update(nodes, nodes_per_bucket: nil, copying_buckets_count: 0)
+      @nodes_per_bucket      = nodes_per_bucket
+      @nodes                 = nodes.first(nodes_per_bucket)
+      @copying_buckets_count = copying_buckets_count
+    end
+
+
+    def nodes
+      @nodes
+    end
+
+    # 由于线上集群可能有上万个数据节点，维护这样的列表也需要
+    # 占用大量的内存，因此这里只保留第一批节点，降低内存占用
+    # TODO: 合并重复的节点，进一步优化内存
+    alias_method :primary_nodes, :nodes
+
+
+    def buckets
+      @bucket ||= [nodes]
+    end
+
+
+  end
+
+end

data/lib/tair/connection.rb

@@ -0,0 +1,56 @@
+require 'stringio'
+require 'socket'
+require 'tair/log'
+
+module Tair
+
+  class Connection
+
+    include Log
+
+    attr_accessor :host, :port, :timeout
+
+
+    def initialize(host, port, opts={})
+      @host, @port = host, port.to_i
+      @opts = opts || {}
+      @timeout = opts[:timeout] || 10.0
+    end
+
+
+    TAIR_CHUNK_SIZE = 1460
+
+    def operate(req, &block)
+
+      socket = TCPSocket.new(host, port)
+      socket.sendmsg req.encode.to_s
+
+      log_time "transporting data", ::Logger::DEBUG do
+        ready = IO.select([socket], nil, nil, timeout)
+
+        if ready
+          Tair::Protocol::ResponseHeader.read(socket.recv(12))
+          socket.recv(4).tap do |len|
+            byte_len = len.unpack('L>').first
+            @response = socket.recv(byte_len)
+          end
+        else
+          raise "Socket communications timed out after #{timeout} seconds"
+        end
+      end
+
+      if block_given?
+        return yield(@response)
+      else
+        return @response
+      end
+
+    ensure
+      socket.close unless socket.closed?
+      @response.clear
+      @response = nil
+    end
+
+  end
+
+end

data/lib/tair/error.rb

@@ -0,0 +1,6 @@
+module Tair
+  class TairError           < StandardError; end
+  class NoAvailableDataNode < TairError; end
+  class InvalidCountStep    < TairError; end
+  class InvalidKeyType      < TairError; end
+end

data/lib/tair/key.rb

@@ -0,0 +1,11 @@
+module Tair
+  module Key
+
+    def valid_key? key
+      return false if key.nil? or key == false or key == true
+      ::Tair::TairObject.from(key).value_len < 100
+    end
+
+  end
+
+end

data/lib/tair/log.rb

@@ -0,0 +1,69 @@
+require 'logger'
+require 'colored'
+
+module Tair
+  module Log
+
+    def self.included(klass)
+      klass.class_eval do
+        attr_writer :logger
+        attr_writer :logger_colorize
+      end
+    end
+
+
+    %w[debug info warn fatal].each do |level|
+      define_method level do |msg=nil, &block|
+        logger && logger.send(level, msg, &block)
+      end
+    end
+
+
+    def log_bytes string
+      # 写入字节到文件，方便与其他语言的SDK做对比
+      File.open(tair_bytes_log_file, 'w') do |f|
+        bytes = string.each_codepoint.to_a
+        debug { bytes.map {|b| b.to_s(16) }.join(" ") }
+        f.write bytes.join(",\n")
+      end
+    end
+
+
+    def tair_bytes_log_file
+      "log/bytes.log"
+    end
+
+
+    def logger
+      return @logger if defined? @logger
+      @logger ||= ::Logger.new(STDOUT).tap do |logger|
+        logger.level = ::Logger::INFO
+        logger.formatter = proc { |serverity, time, prog, msg |
+          colorize("Tair ") << msg.to_s << "\n"
+        }
+      end
+    end
+
+
+    def log_time msg, serverity=::Logger::INFO, &block
+      t = Time.now
+      block.call.tap do |ret|
+        elapsed = '%.3f' % ((Time.now - t) * 1000)
+        logger.log(serverity) { colorize("(#{elapsed}ms)") << " #{msg}" }
+      end
+    end
+
+
+    def colorize text
+      logger_colorize.call(text)
+    end
+
+
+    def logger_colorize
+      @logger_colorize ||= proc { |t|
+        t.to_s.bold
+      }
+    end
+  end
+
+end