stn-dcell 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.coveralls.yml +1 -0
- data/.gitignore +7 -0
- data/.rspec +4 -0
- data/.travis.yml +30 -0
- data/CHANGES.md +53 -0
- data/Gemfile +18 -0
- data/LICENSE.txt +20 -0
- data/README.md +168 -0
- data/Rakefile +4 -0
- data/benchmarks/messaging.rb +73 -0
- data/benchmarks/receiver.rb +37 -0
- data/dcell.gemspec +29 -0
- data/examples/itchy.rb +26 -0
- data/examples/scratchy.rb +12 -0
- data/explorer/css/bootstrap-responsive.css +686 -0
- data/explorer/css/bootstrap-responsive.min.css +12 -0
- data/explorer/css/bootstrap.css +3990 -0
- data/explorer/css/bootstrap.min.css +689 -0
- data/explorer/css/explorer.css +28 -0
- data/explorer/ico/favicon.ico +0 -0
- data/explorer/img/glyphicons-halflings-white.png +0 -0
- data/explorer/img/glyphicons-halflings.png +0 -0
- data/explorer/img/logo.png +0 -0
- data/explorer/index.html.erb +94 -0
- data/explorer/js/bootstrap.js +1726 -0
- data/explorer/js/bootstrap.min.js +6 -0
- data/lib/dcell.rb +127 -0
- data/lib/dcell/actor_proxy.rb +30 -0
- data/lib/dcell/celluloid_ext.rb +120 -0
- data/lib/dcell/directory.rb +31 -0
- data/lib/dcell/explorer.rb +74 -0
- data/lib/dcell/future_proxy.rb +32 -0
- data/lib/dcell/global.rb +23 -0
- data/lib/dcell/info_service.rb +122 -0
- data/lib/dcell/mailbox_proxy.rb +53 -0
- data/lib/dcell/messages.rb +65 -0
- data/lib/dcell/node.rb +138 -0
- data/lib/dcell/node_manager.rb +79 -0
- data/lib/dcell/registries/cassandra_adapter.rb +126 -0
- data/lib/dcell/registries/mongodb_adapter.rb +85 -0
- data/lib/dcell/registries/redis_adapter.rb +95 -0
- data/lib/dcell/registries/zk_adapter.rb +123 -0
- data/lib/dcell/responses.rb +16 -0
- data/lib/dcell/router.rb +46 -0
- data/lib/dcell/rpc.rb +95 -0
- data/lib/dcell/rspec.rb +1 -0
- data/lib/dcell/server.rb +73 -0
- data/lib/dcell/version.rb +3 -0
- data/logo.png +0 -0
- data/spec/dcell/actor_proxy_spec.rb +72 -0
- data/spec/dcell/celluloid_ext_spec.rb +32 -0
- data/spec/dcell/directory_spec.rb +22 -0
- data/spec/dcell/explorer_spec.rb +17 -0
- data/spec/dcell/global_spec.rb +25 -0
- data/spec/dcell/node_spec.rb +23 -0
- data/spec/dcell/registries/mongodb_adapter_spec.rb +7 -0
- data/spec/dcell/registries/redis_adapter_spec.rb +6 -0
- data/spec/dcell/registries/zk_adapter_spec.rb +28 -0
- data/spec/options/01-options.rb +10 -0
- data/spec/options/02-registry.rb +13 -0
- data/spec/spec_helper.rb +28 -0
- data/spec/support/helpers.rb +46 -0
- data/spec/support/registry_examples.rb +35 -0
- data/spec/test_node.rb +38 -0
- data/tasks/cassandra.task +84 -0
- data/tasks/rspec.task +7 -0
- data/tasks/zookeeper.task +58 -0
- metadata +239 -0
@@ -0,0 +1,126 @@
|
|
1
|
+
require 'cassandra'
|
2
|
+
|
3
|
+
# create the keyspace / columnfamily with cqlsh
|
4
|
+
#
|
5
|
+
# create keyspace dcell
|
6
|
+
# with strategy_class='SimpleStrategy'
|
7
|
+
# and strategy_options:replication_factor=3;
|
8
|
+
#
|
9
|
+
# create columnfamily dcell (dcell_type ascii primary key);
|
10
|
+
|
11
|
+
# not sure this is right yet ...
|
12
|
+
# Keyspace "whatever" [
|
13
|
+
# ColumnFamily "dcell" {
|
14
|
+
# RowKey "nodes": {
|
15
|
+
# <nodeid>: <address>,
|
16
|
+
# <nodeid>: <address>,
|
17
|
+
# ...
|
18
|
+
# }
|
19
|
+
# RowKey "globals": {
|
20
|
+
# <key>: <marshal blob>,
|
21
|
+
# <key>: <marshal blob>,
|
22
|
+
# ...
|
23
|
+
# }
|
24
|
+
# }
|
25
|
+
# ]
|
26
|
+
#
|
27
|
+
|
28
|
+
module DCell
|
29
|
+
module Registry
|
30
|
+
class CassandraAdapter
|
31
|
+
DEFAULT_KEYSPACE = "dcell"
|
32
|
+
DEFAULT_CF = "dcell"
|
33
|
+
|
34
|
+
def initialize(options)
|
35
|
+
# Convert all options to symbols :/
|
36
|
+
options = options.inject({}) { |h,(k,v)| h[k.to_sym] = v; h }
|
37
|
+
|
38
|
+
keyspace = options[:keyspace] || DEFAULT_KEYSPACE
|
39
|
+
columnfamily = options[:columnfamily] || DEFAULT_CF
|
40
|
+
|
41
|
+
options[:servers] ||= []
|
42
|
+
options[:servers] << options[:server] if options[:server]
|
43
|
+
options[:servers] << "localhost:9160" unless options[:servers].any?
|
44
|
+
|
45
|
+
cass = Cassandra.new(keyspace, options[:servers])
|
46
|
+
|
47
|
+
@node_registry = NodeRegistry.new(cass, columnfamily)
|
48
|
+
@global_registry = GlobalRegistry.new(cass, columnfamily)
|
49
|
+
end
|
50
|
+
|
51
|
+
def remove_node(node)
|
52
|
+
@node_registry.remove node
|
53
|
+
end
|
54
|
+
|
55
|
+
def clear_all_nodes
|
56
|
+
@node_registry.clear_all
|
57
|
+
end
|
58
|
+
|
59
|
+
def clear_globals
|
60
|
+
@global_registry.clear_all
|
61
|
+
end
|
62
|
+
|
63
|
+
class NodeRegistry
|
64
|
+
def initialize(cass, cf)
|
65
|
+
@cass = cass
|
66
|
+
@cf = cf
|
67
|
+
end
|
68
|
+
|
69
|
+
def get(node_id)
|
70
|
+
@cass.get @cf, "nodes", node_id
|
71
|
+
end
|
72
|
+
|
73
|
+
def set(node_id, addr)
|
74
|
+
@cass.insert @cf, "nodes", { node_id => addr }
|
75
|
+
end
|
76
|
+
|
77
|
+
def nodes
|
78
|
+
@cass.get(@cf, "nodes").keys
|
79
|
+
end
|
80
|
+
|
81
|
+
def remove(node)
|
82
|
+
@cass.remove @cf, "nodes", { node_id => node }
|
83
|
+
end
|
84
|
+
|
85
|
+
def clear_all
|
86
|
+
@cass.del @cf, "nodes"
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def get_node(node_id); @node_registry.get(node_id) end
|
91
|
+
def set_node(node_id, addr); @node_registry.set(node_id, addr) end
|
92
|
+
def nodes; @node_registry.nodes end
|
93
|
+
|
94
|
+
class GlobalRegistry
|
95
|
+
def initialize(cass, cf)
|
96
|
+
@cass = cass
|
97
|
+
@cf = cf
|
98
|
+
end
|
99
|
+
|
100
|
+
def get(key)
|
101
|
+
string = @cass.get @cf, "globals", key.to_s
|
102
|
+
Marshal.load string if string
|
103
|
+
end
|
104
|
+
|
105
|
+
# Set a global value
|
106
|
+
def set(key, value)
|
107
|
+
string = Marshal.dump value
|
108
|
+
@cass.insert @cf, "globals", { key.to_s => string }
|
109
|
+
end
|
110
|
+
|
111
|
+
# The keys to all globals in the system
|
112
|
+
def global_keys
|
113
|
+
@cass.get(@cf, "globals").keys
|
114
|
+
end
|
115
|
+
|
116
|
+
def clear_all
|
117
|
+
@cass.del @cf, "globals"
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def get_global(key); @global_registry.get(key) end
|
122
|
+
def set_global(key, value); @global_registry.set(key, value) end
|
123
|
+
def global_keys; @global_registry.global_keys end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'mongoid'
|
2
|
+
|
3
|
+
module DCell
|
4
|
+
module Registry
|
5
|
+
class MongodbAdapter
|
6
|
+
# Setup connection to mongodb
|
7
|
+
# config: path to mongoid configuration file
|
8
|
+
# env: mongoid environment to use
|
9
|
+
def initialize(options)
|
10
|
+
if options[:config]
|
11
|
+
Mongoid.load! options[:config], options[:env]
|
12
|
+
elsif options[:db]
|
13
|
+
Mongoid.connect_to options[:db]
|
14
|
+
end
|
15
|
+
if options[:options]
|
16
|
+
Mongoid.options = options[:options]
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class DCellNode
|
21
|
+
include Mongoid::Document
|
22
|
+
|
23
|
+
field :key, type: String
|
24
|
+
field :value, type: Hash
|
25
|
+
end
|
26
|
+
|
27
|
+
class DCellGlobal
|
28
|
+
include Mongoid::Document
|
29
|
+
|
30
|
+
field :key, type: String
|
31
|
+
field :value, type: Hash
|
32
|
+
end
|
33
|
+
|
34
|
+
class Proxy
|
35
|
+
class << self
|
36
|
+
def set(storage, key, value)
|
37
|
+
entry = storage.find_or_create_by(key: key)
|
38
|
+
entry.value = {'v' => value}
|
39
|
+
entry.save!
|
40
|
+
value
|
41
|
+
end
|
42
|
+
|
43
|
+
def get(storage, key)
|
44
|
+
first = storage.where(key: key).first
|
45
|
+
if first and first.value
|
46
|
+
return first.value['v']
|
47
|
+
end
|
48
|
+
nil
|
49
|
+
end
|
50
|
+
|
51
|
+
def all(storage)
|
52
|
+
keys = []
|
53
|
+
storage.each do |entry|
|
54
|
+
keys << entry.key
|
55
|
+
end
|
56
|
+
keys
|
57
|
+
end
|
58
|
+
|
59
|
+
def remove(storage, key)
|
60
|
+
begin
|
61
|
+
storage.where(key: key).delete
|
62
|
+
rescue
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def clear_all(storage)
|
67
|
+
storage.delete_all
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def get_node(node_id); Proxy.get(DCellNode, node_id) end
|
73
|
+
def set_node(node_id, addr); Proxy.set(DCellNode, node_id, addr) end
|
74
|
+
def nodes; Proxy.all(DCellNode) end
|
75
|
+
def remove_node(node_id); Proxy.remove(DCellNode, node_id) end
|
76
|
+
def clear_all_nodes; Proxy.clear_all(DCellNode) end
|
77
|
+
|
78
|
+
def get_global(key); Proxy.get(DCellGlobal, key) end
|
79
|
+
def set_global(key, value); Proxy.set(DCellGlobal, key, value) end
|
80
|
+
def global_keys; Proxy.all(DCellGlobal) end
|
81
|
+
def clear_globals; Proxy.clear_all(DCellGlobal) end
|
82
|
+
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'celluloid/redis'
|
2
|
+
require 'redis/connection/celluloid'
|
3
|
+
require 'redis-namespace'
|
4
|
+
|
5
|
+
module DCell
|
6
|
+
module Registry
|
7
|
+
class RedisAdapter
|
8
|
+
def initialize(options)
|
9
|
+
# Convert all options to symbols :/
|
10
|
+
options = options.inject({}) { |h,(k,v)| h[k.to_sym] = v; h }
|
11
|
+
|
12
|
+
@env = options[:env] || 'production'
|
13
|
+
@namespace = options[:namespace] || "dcell_#{@env}"
|
14
|
+
|
15
|
+
redis = Redis.new options
|
16
|
+
@redis = Redis::Namespace.new @namespace, :redis => redis
|
17
|
+
|
18
|
+
@node_registry = NodeRegistry.new(@redis)
|
19
|
+
@global_registry = GlobalRegistry.new(@redis)
|
20
|
+
end
|
21
|
+
|
22
|
+
def remove_node(node)
|
23
|
+
@node_registry.remove node
|
24
|
+
end
|
25
|
+
|
26
|
+
def clear_all_nodes
|
27
|
+
@node_registry.clear_all
|
28
|
+
end
|
29
|
+
|
30
|
+
def clear_globals
|
31
|
+
@global_registry.clear_all
|
32
|
+
end
|
33
|
+
|
34
|
+
class NodeRegistry
|
35
|
+
def initialize(redis)
|
36
|
+
@redis = redis
|
37
|
+
end
|
38
|
+
|
39
|
+
def get(node_id)
|
40
|
+
@redis.hget 'nodes', node_id
|
41
|
+
end
|
42
|
+
|
43
|
+
def set(node_id, addr)
|
44
|
+
@redis.hset 'nodes', node_id, addr
|
45
|
+
end
|
46
|
+
|
47
|
+
def nodes
|
48
|
+
@redis.hkeys 'nodes'
|
49
|
+
end
|
50
|
+
|
51
|
+
def remove(node)
|
52
|
+
@redis.hdel 'nodes', node
|
53
|
+
end
|
54
|
+
|
55
|
+
def clear_all
|
56
|
+
@redis.del 'nodes'
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def get_node(node_id); @node_registry.get(node_id) end
|
61
|
+
def set_node(node_id, addr); @node_registry.set(node_id, addr) end
|
62
|
+
def nodes; @node_registry.nodes end
|
63
|
+
|
64
|
+
class GlobalRegistry
|
65
|
+
def initialize(redis)
|
66
|
+
@redis = redis
|
67
|
+
end
|
68
|
+
|
69
|
+
def get(key)
|
70
|
+
string = @redis.hget 'globals', key.to_s
|
71
|
+
Marshal.load string if string
|
72
|
+
end
|
73
|
+
|
74
|
+
# Set a global value
|
75
|
+
def set(key, value)
|
76
|
+
string = Marshal.dump value
|
77
|
+
@redis.hset 'globals', key.to_s, string
|
78
|
+
end
|
79
|
+
|
80
|
+
# The keys to all globals in the system
|
81
|
+
def global_keys
|
82
|
+
@redis.hkeys 'globals'
|
83
|
+
end
|
84
|
+
|
85
|
+
def clear_all
|
86
|
+
@redis.del 'globals'
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def get_global(key); @global_registry.get(key) end
|
91
|
+
def set_global(key, value); @global_registry.set(key, value) end
|
92
|
+
def global_keys; @global_registry.global_keys end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
require 'zk'
|
2
|
+
|
3
|
+
module DCell
|
4
|
+
module Registry
|
5
|
+
class ZkAdapter
|
6
|
+
PREFIX = "/dcell"
|
7
|
+
DEFAULT_PORT = 2181
|
8
|
+
|
9
|
+
# Create a new connection to Zookeeper
|
10
|
+
#
|
11
|
+
# servers: a list of Zookeeper servers to connect to. Each server in the
|
12
|
+
# list has a host/port configuration
|
13
|
+
def initialize(options)
|
14
|
+
# Stringify keys :/
|
15
|
+
options = options.inject({}) { |h,(k,v)| h[k.to_s] = v; h }
|
16
|
+
|
17
|
+
@env = options['env'] || 'production'
|
18
|
+
@base_path = "#{PREFIX}/#{@env}"
|
19
|
+
|
20
|
+
# Let them specify a single server instead of many
|
21
|
+
server = options['server']
|
22
|
+
if server
|
23
|
+
servers = [server]
|
24
|
+
else
|
25
|
+
servers = options['servers']
|
26
|
+
raise "no Zookeeper servers given" unless servers
|
27
|
+
end
|
28
|
+
|
29
|
+
# Add the default Zookeeper port unless specified
|
30
|
+
servers.map! do |server|
|
31
|
+
if server[/:\d+$/]
|
32
|
+
server
|
33
|
+
else
|
34
|
+
"#{server}:#{DEFAULT_PORT}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
@zk = ZK.new(*servers)
|
39
|
+
@node_registry = Registry.new(@zk, @base_path, :nodes, true)
|
40
|
+
@global_registry = Registry.new(@zk, @base_path, :globals, false)
|
41
|
+
end
|
42
|
+
|
43
|
+
class Registry
|
44
|
+
def initialize(zk, base_path, name, ephemeral)
|
45
|
+
@zk = zk
|
46
|
+
@base_path = File.join(base_path, name.to_s)
|
47
|
+
@ephemeral = ephemeral
|
48
|
+
@zk.mkdir_p @base_path
|
49
|
+
@events = {}
|
50
|
+
end
|
51
|
+
|
52
|
+
def register(key)
|
53
|
+
path = "#{@base_path}/#{key}"
|
54
|
+
@events[path] ||= @zk.register(path) do |event|
|
55
|
+
key = event.path.match(/#{@base_path}\/(\w+)/)[1]
|
56
|
+
@events[event.path].unsubscribe
|
57
|
+
@events[event.path] = nil
|
58
|
+
if event.node_changed?
|
59
|
+
Celluloid::logger.debug "zk callback: node changed!"
|
60
|
+
Node.update(key)
|
61
|
+
end
|
62
|
+
if event.node_deleted?
|
63
|
+
Celluloid::logger.debug "zk callback: node deleted!"
|
64
|
+
Node.remove(key)
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def get(key)
|
71
|
+
register(key)
|
72
|
+
result, _ = @zk.get("#{@base_path}/#{key}", watch: true)
|
73
|
+
Marshal.load result
|
74
|
+
rescue ZK::Exceptions::NoNode
|
75
|
+
end
|
76
|
+
|
77
|
+
def set(key, value)
|
78
|
+
register(key)
|
79
|
+
path = "#{@base_path}/#{key}"
|
80
|
+
string = Marshal.dump value
|
81
|
+
@zk.set path, string
|
82
|
+
rescue ZK::Exceptions::NoNode
|
83
|
+
@zk.create path, string, :ephemeral => @ephemeral
|
84
|
+
end
|
85
|
+
|
86
|
+
def all
|
87
|
+
@zk.children @base_path
|
88
|
+
end
|
89
|
+
|
90
|
+
def remove(key)
|
91
|
+
keys.each do |key|
|
92
|
+
path = "#{@base_path}/#{key}"
|
93
|
+
@zk.delete path
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def clear_all
|
98
|
+
# delete znodes so any registered
|
99
|
+
# callback is triggered
|
100
|
+
all.each do |key|
|
101
|
+
remove key
|
102
|
+
end
|
103
|
+
@events.values.compact.each(&:unsubscribe)
|
104
|
+
@events.clear
|
105
|
+
@zk.rm_rf @base_path
|
106
|
+
@zk.mkdir_p @base_path
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def get_node(node_id); @node_registry.get(node_id) end
|
111
|
+
def set_node(node_id, addr); @node_registry.set(node_id, addr) end
|
112
|
+
def nodes; @node_registry.all end
|
113
|
+
def remove_node(node_id); @node_registry.clear(node_id) end
|
114
|
+
def clear_all_nodes; @node_registry.clear_all end
|
115
|
+
|
116
|
+
def get_global(key); @global_registry.get(key) end
|
117
|
+
def set_global(key, value); @global_registry.set(key, value) end
|
118
|
+
def global_keys; @global_registry.all end
|
119
|
+
def clear_globals; @global_registry.clear_all end
|
120
|
+
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module DCell
|
2
|
+
# Responses to calls
|
3
|
+
class Response
|
4
|
+
attr_reader :request_id, :value
|
5
|
+
|
6
|
+
def initialize(request_id, value)
|
7
|
+
@request_id, @value = request_id, value
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
# Request successful
|
12
|
+
class SuccessResponse < Response; end
|
13
|
+
|
14
|
+
# Request failed
|
15
|
+
class ErrorResponse < Response; end
|
16
|
+
end
|