tair 0.1.0.pre
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.rspec +2 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/Rakefile +6 -0
- data/ext/tair/extconf.rb +3 -0
- data/ext/tair/tair_murmurhash.cpp +72 -0
- data/lib/tair.rb +8 -0
- data/lib/tair/client.rb +107 -0
- data/lib/tair/cluster.rb +51 -0
- data/lib/tair/connection.rb +56 -0
- data/lib/tair/error.rb +6 -0
- data/lib/tair/key.rb +11 -0
- data/lib/tair/log.rb +69 -0
- data/lib/tair/operation/count.rb +12 -0
- data/lib/tair/operation/decr.rb +12 -0
- data/lib/tair/operation/delete.rb +49 -0
- data/lib/tair/operation/fetch_data_servers.rb +123 -0
- data/lib/tair/operation/get.rb +128 -0
- data/lib/tair/operation/incr.rb +76 -0
- data/lib/tair/operation/put.rb +168 -0
- data/lib/tair/protocol.rb +5 -0
- data/lib/tair/protocol/key_meta.rb +9 -0
- data/lib/tair/protocol/murmurhash.rb +11 -0
- data/lib/tair/protocol/response_header.rb +14 -0
- data/lib/tair/protocol/socket_signature.rb +12 -0
- data/lib/tair/protocol/tair_object.rb +67 -0
- data/lib/tair/request.rb +70 -0
- data/lib/tair/response.rb +19 -0
- data/lib/tair/version.rb +12 -0
- data/log/.gitkeep +0 -0
- data/tair.gemspec +34 -0
- metadata +189 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 02165903d5bd1c58ab4b7d9f005c8d60d2010f2a
|
4
|
+
data.tar.gz: cdcae2a4411765d813550b97e868e10af5bc0291
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6bf80ccb4622346a4aafb25e439799a90792356a4091304124f270853614c1a3476473cb3be546ae707c634c532a71deb2acce9e7167c05c20aa837e5b574394
|
7
|
+
data.tar.gz: b6f4d76b33be133a1834db4402df6c9d22326a4a99f0caee953eb9135225d7c3b17ddc1643e21cdee4f5bd0d026151cc5a23427114b7d5d27d24cac05573f82c
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 如彼
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
data/ext/tair/extconf.rb
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
#include "rice/Module.hpp"
|
2
|
+
#include "rice/Class.hpp"
|
3
|
+
#include "rice/String.hpp"
|
4
|
+
|
5
|
+
using namespace Rice;
|
6
|
+
using namespace std;
|
7
|
+
|
8
|
+
#define MURMURHASH_M 0x5bd1e995
|
9
|
+
|
10
|
+
uint64_t digest(string str, uint32_t seed)
|
11
|
+
{
|
12
|
+
const char *key = str.c_str();
|
13
|
+
int len = str.length();
|
14
|
+
uint32_t h = seed ^ len;
|
15
|
+
int index = 0;
|
16
|
+
|
17
|
+
while (len >= 4) {
|
18
|
+
uint32_t k = (key[index] & 0xff) | ((key[index + 1] << 8) & 0xff00)
|
19
|
+
| ((key[index + 2] << 16) & 0xff0000)
|
20
|
+
| (key[index + 3] << 24);
|
21
|
+
|
22
|
+
k *= MURMURHASH_M;
|
23
|
+
k ^= (k >> 24);
|
24
|
+
k *= MURMURHASH_M;
|
25
|
+
h *= MURMURHASH_M;
|
26
|
+
h ^= k;
|
27
|
+
index += 4;
|
28
|
+
len -= 4;
|
29
|
+
}
|
30
|
+
|
31
|
+
switch (len) {
|
32
|
+
case 3:
|
33
|
+
h ^= (key[index + 2] << 16);
|
34
|
+
|
35
|
+
case 2:
|
36
|
+
h ^= (key[index + 1] << 8);
|
37
|
+
|
38
|
+
case 1:
|
39
|
+
h ^= key[index];
|
40
|
+
h *= MURMURHASH_M;
|
41
|
+
}
|
42
|
+
|
43
|
+
h ^= (h >> 13);
|
44
|
+
h *= MURMURHASH_M;
|
45
|
+
h ^= (h >> 15);
|
46
|
+
return h;
|
47
|
+
}
|
48
|
+
|
49
|
+
|
50
|
+
Object tmh_initialize(Object self, String str, uint32_t seed)
|
51
|
+
{
|
52
|
+
self.iv_set("@string", str);
|
53
|
+
self.iv_set("@seed", seed);
|
54
|
+
}
|
55
|
+
|
56
|
+
Object tmh_digest(Object self)
|
57
|
+
{
|
58
|
+
string str = self.iv_get("@string").to_s().str();
|
59
|
+
uint32_t seed = from_ruby<uint32_t>(self.iv_get("@seed"));
|
60
|
+
uint32_t hash = digest(str, seed);
|
61
|
+
return to_ruby<uint32_t>(hash);
|
62
|
+
}
|
63
|
+
|
64
|
+
extern "C"
|
65
|
+
void Init_tair_murmurhash()
|
66
|
+
{
|
67
|
+
Module rb_mTair = define_module("Tair");
|
68
|
+
Class rb_cTairMurmurhash =
|
69
|
+
define_class_under(rb_mTair, "TairMurmurhash")
|
70
|
+
.define_method( "initialize", &tmh_initialize )
|
71
|
+
.define_method( "digest", &tmh_digest );
|
72
|
+
}
|
data/lib/tair.rb
ADDED
data/lib/tair/client.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'tair/cluster'
|
2
|
+
require 'tair/key'
|
3
|
+
require 'tair/request'
|
4
|
+
require 'tair/response'
|
5
|
+
require 'tair/connection'
|
6
|
+
require 'tair/protocol/murmurhash'
|
7
|
+
|
8
|
+
Dir.glob(File.expand_path '../operation/*.rb', __FILE__) do |f|
|
9
|
+
require f
|
10
|
+
end
|
11
|
+
|
12
|
+
module Tair
|
13
|
+
|
14
|
+
class Client
|
15
|
+
|
16
|
+
attr_accessor :namespace
|
17
|
+
|
18
|
+
include Operation
|
19
|
+
include Log
|
20
|
+
|
21
|
+
def self.create(host, port, opts={}, &block)
|
22
|
+
new(host, port, opts).tap do |client|
|
23
|
+
block && block.call(client)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
attr_reader :host, :port, :group
|
29
|
+
attr_writer :data_servers
|
30
|
+
|
31
|
+
def initialize(host, port, group: nil, namespace: 0xab)
|
32
|
+
@host, @port, @group, @namespace = host, port, group, namespace
|
33
|
+
end
|
34
|
+
|
35
|
+
|
36
|
+
# 配置有变化时,需要更新到客户端
|
37
|
+
def watch_cluster_config(interval=120)
|
38
|
+
Thread.new { watch_cluster_config!(interval) }
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
def watch_cluster_config!(interval=120)
|
43
|
+
while true
|
44
|
+
sleep interval
|
45
|
+
@cluster = fetch_data_servers(group)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
WRITE_OPERATIONS = [:put, :incr, :set_count, :delete]
|
51
|
+
|
52
|
+
def operate(op, request, key: nil, &block)
|
53
|
+
request.namespace = namespace
|
54
|
+
will_write_data = WRITE_OPERATIONS.include? op
|
55
|
+
conn = data_conn(key, will_write_data: will_write_data)
|
56
|
+
log_time "#{op} #{key.inspect} on #{conn.host}" do
|
57
|
+
conn.operate(request, &block)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
|
62
|
+
def init_cluster(retries: 5)
|
63
|
+
count = retries
|
64
|
+
begin
|
65
|
+
cluster
|
66
|
+
rescue => e
|
67
|
+
error { "init cluster error: #{e}" }
|
68
|
+
count -= 1
|
69
|
+
if count >= 0
|
70
|
+
retry
|
71
|
+
else
|
72
|
+
raise
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
# 目前读写对应的节点都是一样的,以后
|
81
|
+
# 支持 down_servers 配置时,就可能不
|
82
|
+
# 同
|
83
|
+
def data_conn(key, will_write_data: false)
|
84
|
+
node = cluster.data_node_for(key, will_write_data: will_write_data)
|
85
|
+
Connection.new(*node.split(':')).tap do |cn|
|
86
|
+
cn.logger = logger
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
alias_method :conn, :data_conn
|
91
|
+
|
92
|
+
|
93
|
+
def cluster
|
94
|
+
@cluster ||= fetch_data_servers(group)
|
95
|
+
end
|
96
|
+
|
97
|
+
|
98
|
+
def config_conn
|
99
|
+
opts = {}
|
100
|
+
@config_conn ||= Connection.new(host, port, opts)
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
|
105
|
+
end
|
106
|
+
|
107
|
+
end
|
data/lib/tair/cluster.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
module Tair
|
2
|
+
|
3
|
+
# 一个 cluster 是一个 tair 节点组合
|
4
|
+
# cluster 由 bucket 构成
|
5
|
+
# bucket 由 data_node 构成
|
6
|
+
# node -> bucket -> cluster
|
7
|
+
class Cluster
|
8
|
+
|
9
|
+
attr_reader :nodes_per_bucket, :copying_buckets_count
|
10
|
+
|
11
|
+
|
12
|
+
# 根据指定的值,找到对应的数据节点
|
13
|
+
def data_node_for key, will_write_data: false
|
14
|
+
tair_key = Tair::TairObject.from(key).to_binary_s[4..-1]
|
15
|
+
digest = Tair::Protocol::Murmurhash.digest tair_key, 97
|
16
|
+
idx = digest % nodes_per_bucket
|
17
|
+
primary_nodes[idx]
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
# 更新集群信息
|
22
|
+
#
|
23
|
+
# Params:
|
24
|
+
# - nodes Array 所有节点数组
|
25
|
+
# - nodes_per_bucket Integer 多少个节点构成一个 bucket
|
26
|
+
# - bucket_for_copy Integer 有多少个 bucket 用来作备库
|
27
|
+
def update(nodes, nodes_per_bucket: nil, copying_buckets_count: 0)
|
28
|
+
@nodes_per_bucket = nodes_per_bucket
|
29
|
+
@nodes = nodes.first(nodes_per_bucket)
|
30
|
+
@copying_buckets_count = copying_buckets_count
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
def nodes
|
35
|
+
@nodes
|
36
|
+
end
|
37
|
+
|
38
|
+
# 由于线上集群可能有上万个数据节点,维护这样的列表也需要
|
39
|
+
# 占用大量的内存,因此这里只保留第一批节点,降低内存占用
|
40
|
+
# TODO: 合并重复的节点,进一步优化内存
|
41
|
+
alias_method :primary_nodes, :nodes
|
42
|
+
|
43
|
+
|
44
|
+
def buckets
|
45
|
+
@bucket ||= [nodes]
|
46
|
+
end
|
47
|
+
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'stringio'
|
2
|
+
require 'socket'
|
3
|
+
require 'tair/log'
|
4
|
+
|
5
|
+
module Tair
|
6
|
+
|
7
|
+
class Connection
|
8
|
+
|
9
|
+
include Log
|
10
|
+
|
11
|
+
attr_accessor :host, :port, :timeout
|
12
|
+
|
13
|
+
|
14
|
+
def initialize(host, port, opts={})
|
15
|
+
@host, @port = host, port.to_i
|
16
|
+
@opts = opts || {}
|
17
|
+
@timeout = opts[:timeout] || 10.0
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
TAIR_CHUNK_SIZE = 1460
|
22
|
+
|
23
|
+
def operate(req, &block)
|
24
|
+
|
25
|
+
socket = TCPSocket.new(host, port)
|
26
|
+
socket.sendmsg req.encode.to_s
|
27
|
+
|
28
|
+
log_time "transporting data", ::Logger::DEBUG do
|
29
|
+
ready = IO.select([socket], nil, nil, timeout)
|
30
|
+
|
31
|
+
if ready
|
32
|
+
Tair::Protocol::ResponseHeader.read(socket.recv(12))
|
33
|
+
socket.recv(4).tap do |len|
|
34
|
+
byte_len = len.unpack('L>').first
|
35
|
+
@response = socket.recv(byte_len)
|
36
|
+
end
|
37
|
+
else
|
38
|
+
raise "Socket communications timed out after #{timeout} seconds"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
if block_given?
|
43
|
+
return yield(@response)
|
44
|
+
else
|
45
|
+
return @response
|
46
|
+
end
|
47
|
+
|
48
|
+
ensure
|
49
|
+
socket.close unless socket.closed?
|
50
|
+
@response.clear
|
51
|
+
@response = nil
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
data/lib/tair/error.rb
ADDED
data/lib/tair/key.rb
ADDED
data/lib/tair/log.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'colored'
|
3
|
+
|
4
|
+
module Tair
|
5
|
+
module Log
|
6
|
+
|
7
|
+
def self.included(klass)
|
8
|
+
klass.class_eval do
|
9
|
+
attr_writer :logger
|
10
|
+
attr_writer :logger_colorize
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
%w[debug info warn fatal].each do |level|
|
16
|
+
define_method level do |msg=nil, &block|
|
17
|
+
logger && logger.send(level, msg, &block)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
def log_bytes string
|
23
|
+
# 写入字节到文件,方便与其他语言的SDK做对比
|
24
|
+
File.open(tair_bytes_log_file, 'w') do |f|
|
25
|
+
bytes = string.each_codepoint.to_a
|
26
|
+
debug { bytes.map {|b| b.to_s(16) }.join(" ") }
|
27
|
+
f.write bytes.join(",\n")
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
def tair_bytes_log_file
|
33
|
+
"log/bytes.log"
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
def logger
|
38
|
+
return @logger if defined? @logger
|
39
|
+
@logger ||= ::Logger.new(STDOUT).tap do |logger|
|
40
|
+
logger.level = ::Logger::INFO
|
41
|
+
logger.formatter = proc { |serverity, time, prog, msg |
|
42
|
+
colorize("Tair ") << msg.to_s << "\n"
|
43
|
+
}
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
def log_time msg, serverity=::Logger::INFO, &block
|
49
|
+
t = Time.now
|
50
|
+
block.call.tap do |ret|
|
51
|
+
elapsed = '%.3f' % ((Time.now - t) * 1000)
|
52
|
+
logger.log(serverity) { colorize("(#{elapsed}ms)") << " #{msg}" }
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
|
57
|
+
def colorize text
|
58
|
+
logger_colorize.call(text)
|
59
|
+
end
|
60
|
+
|
61
|
+
|
62
|
+
def logger_colorize
|
63
|
+
@logger_colorize ||= proc { |t|
|
64
|
+
t.to_s.bold
|
65
|
+
}
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|