stn-dcell 0.16.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (69) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +1 -0
  3. data/.gitignore +7 -0
  4. data/.rspec +4 -0
  5. data/.travis.yml +30 -0
  6. data/CHANGES.md +53 -0
  7. data/Gemfile +18 -0
  8. data/LICENSE.txt +20 -0
  9. data/README.md +168 -0
  10. data/Rakefile +4 -0
  11. data/benchmarks/messaging.rb +73 -0
  12. data/benchmarks/receiver.rb +37 -0
  13. data/dcell.gemspec +29 -0
  14. data/examples/itchy.rb +26 -0
  15. data/examples/scratchy.rb +12 -0
  16. data/explorer/css/bootstrap-responsive.css +686 -0
  17. data/explorer/css/bootstrap-responsive.min.css +12 -0
  18. data/explorer/css/bootstrap.css +3990 -0
  19. data/explorer/css/bootstrap.min.css +689 -0
  20. data/explorer/css/explorer.css +28 -0
  21. data/explorer/ico/favicon.ico +0 -0
  22. data/explorer/img/glyphicons-halflings-white.png +0 -0
  23. data/explorer/img/glyphicons-halflings.png +0 -0
  24. data/explorer/img/logo.png +0 -0
  25. data/explorer/index.html.erb +94 -0
  26. data/explorer/js/bootstrap.js +1726 -0
  27. data/explorer/js/bootstrap.min.js +6 -0
  28. data/lib/dcell.rb +127 -0
  29. data/lib/dcell/actor_proxy.rb +30 -0
  30. data/lib/dcell/celluloid_ext.rb +120 -0
  31. data/lib/dcell/directory.rb +31 -0
  32. data/lib/dcell/explorer.rb +74 -0
  33. data/lib/dcell/future_proxy.rb +32 -0
  34. data/lib/dcell/global.rb +23 -0
  35. data/lib/dcell/info_service.rb +122 -0
  36. data/lib/dcell/mailbox_proxy.rb +53 -0
  37. data/lib/dcell/messages.rb +65 -0
  38. data/lib/dcell/node.rb +138 -0
  39. data/lib/dcell/node_manager.rb +79 -0
  40. data/lib/dcell/registries/cassandra_adapter.rb +126 -0
  41. data/lib/dcell/registries/mongodb_adapter.rb +85 -0
  42. data/lib/dcell/registries/redis_adapter.rb +95 -0
  43. data/lib/dcell/registries/zk_adapter.rb +123 -0
  44. data/lib/dcell/responses.rb +16 -0
  45. data/lib/dcell/router.rb +46 -0
  46. data/lib/dcell/rpc.rb +95 -0
  47. data/lib/dcell/rspec.rb +1 -0
  48. data/lib/dcell/server.rb +73 -0
  49. data/lib/dcell/version.rb +3 -0
  50. data/logo.png +0 -0
  51. data/spec/dcell/actor_proxy_spec.rb +72 -0
  52. data/spec/dcell/celluloid_ext_spec.rb +32 -0
  53. data/spec/dcell/directory_spec.rb +22 -0
  54. data/spec/dcell/explorer_spec.rb +17 -0
  55. data/spec/dcell/global_spec.rb +25 -0
  56. data/spec/dcell/node_spec.rb +23 -0
  57. data/spec/dcell/registries/mongodb_adapter_spec.rb +7 -0
  58. data/spec/dcell/registries/redis_adapter_spec.rb +6 -0
  59. data/spec/dcell/registries/zk_adapter_spec.rb +28 -0
  60. data/spec/options/01-options.rb +10 -0
  61. data/spec/options/02-registry.rb +13 -0
  62. data/spec/spec_helper.rb +28 -0
  63. data/spec/support/helpers.rb +46 -0
  64. data/spec/support/registry_examples.rb +35 -0
  65. data/spec/test_node.rb +38 -0
  66. data/tasks/cassandra.task +84 -0
  67. data/tasks/rspec.task +7 -0
  68. data/tasks/zookeeper.task +58 -0
  69. metadata +239 -0
@@ -0,0 +1,126 @@
1
+ require 'cassandra'
2
+
3
+ # create the keyspace / columnfamily with cqlsh
4
+ #
5
+ # create keyspace dcell
6
+ # with strategy_class='SimpleStrategy'
7
+ # and strategy_options:replication_factor=3;
8
+ #
9
+ # create columnfamily dcell (dcell_type ascii primary key);
10
+
11
+ # not sure this is right yet ...
12
+ # Keyspace "whatever" [
13
+ # ColumnFamily "dcell" {
14
+ # RowKey "nodes": {
15
+ # <nodeid>: <address>,
16
+ # <nodeid>: <address>,
17
+ # ...
18
+ # }
19
+ # RowKey "globals": {
20
+ # <key>: <marshal blob>,
21
+ # <key>: <marshal blob>,
22
+ # ...
23
+ # }
24
+ # }
25
+ # ]
26
+ #
27
+
28
+ module DCell
29
+ module Registry
30
+ class CassandraAdapter
31
+ DEFAULT_KEYSPACE = "dcell"
32
+ DEFAULT_CF = "dcell"
33
+
34
+ def initialize(options)
35
+ # Convert all options to symbols :/
36
+ options = options.inject({}) { |h,(k,v)| h[k.to_sym] = v; h }
37
+
38
+ keyspace = options[:keyspace] || DEFAULT_KEYSPACE
39
+ columnfamily = options[:columnfamily] || DEFAULT_CF
40
+
41
+ options[:servers] ||= []
42
+ options[:servers] << options[:server] if options[:server]
43
+ options[:servers] << "localhost:9160" unless options[:servers].any?
44
+
45
+ cass = Cassandra.new(keyspace, options[:servers])
46
+
47
+ @node_registry = NodeRegistry.new(cass, columnfamily)
48
+ @global_registry = GlobalRegistry.new(cass, columnfamily)
49
+ end
50
+
51
+ def remove_node(node)
52
+ @node_registry.remove node
53
+ end
54
+
55
+ def clear_all_nodes
56
+ @node_registry.clear_all
57
+ end
58
+
59
+ def clear_globals
60
+ @global_registry.clear_all
61
+ end
62
+
63
+ class NodeRegistry
64
+ def initialize(cass, cf)
65
+ @cass = cass
66
+ @cf = cf
67
+ end
68
+
69
+ def get(node_id)
70
+ @cass.get @cf, "nodes", node_id
71
+ end
72
+
73
+ def set(node_id, addr)
74
+ @cass.insert @cf, "nodes", { node_id => addr }
75
+ end
76
+
77
+ def nodes
78
+ @cass.get(@cf, "nodes").keys
79
+ end
80
+
81
+ def remove(node)
82
+ @cass.remove @cf, "nodes", { node_id => node }
83
+ end
84
+
85
+ def clear_all
86
+ @cass.del @cf, "nodes"
87
+ end
88
+ end
89
+
90
+ def get_node(node_id); @node_registry.get(node_id) end
91
+ def set_node(node_id, addr); @node_registry.set(node_id, addr) end
92
+ def nodes; @node_registry.nodes end
93
+
94
+ class GlobalRegistry
95
+ def initialize(cass, cf)
96
+ @cass = cass
97
+ @cf = cf
98
+ end
99
+
100
+ def get(key)
101
+ string = @cass.get @cf, "globals", key.to_s
102
+ Marshal.load string if string
103
+ end
104
+
105
+ # Set a global value
106
+ def set(key, value)
107
+ string = Marshal.dump value
108
+ @cass.insert @cf, "globals", { key.to_s => string }
109
+ end
110
+
111
+ # The keys to all globals in the system
112
+ def global_keys
113
+ @cass.get(@cf, "globals").keys
114
+ end
115
+
116
+ def clear_all
117
+ @cass.del @cf, "globals"
118
+ end
119
+ end
120
+
121
+ def get_global(key); @global_registry.get(key) end
122
+ def set_global(key, value); @global_registry.set(key, value) end
123
+ def global_keys; @global_registry.global_keys end
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,85 @@
1
+ require 'mongoid'
2
+
3
+ module DCell
4
+ module Registry
5
+ class MongodbAdapter
6
+ # Setup connection to mongodb
7
+ # config: path to mongoid configuration file
8
+ # env: mongoid environment to use
9
+ def initialize(options)
10
+ if options[:config]
11
+ Mongoid.load! options[:config], options[:env]
12
+ elsif options[:db]
13
+ Mongoid.connect_to options[:db]
14
+ end
15
+ if options[:options]
16
+ Mongoid.options = options[:options]
17
+ end
18
+ end
19
+
20
+ class DCellNode
21
+ include Mongoid::Document
22
+
23
+ field :key, type: String
24
+ field :value, type: Hash
25
+ end
26
+
27
+ class DCellGlobal
28
+ include Mongoid::Document
29
+
30
+ field :key, type: String
31
+ field :value, type: Hash
32
+ end
33
+
34
+ class Proxy
35
+ class << self
36
+ def set(storage, key, value)
37
+ entry = storage.find_or_create_by(key: key)
38
+ entry.value = {'v' => value}
39
+ entry.save!
40
+ value
41
+ end
42
+
43
+ def get(storage, key)
44
+ first = storage.where(key: key).first
45
+ if first and first.value
46
+ return first.value['v']
47
+ end
48
+ nil
49
+ end
50
+
51
+ def all(storage)
52
+ keys = []
53
+ storage.each do |entry|
54
+ keys << entry.key
55
+ end
56
+ keys
57
+ end
58
+
59
+ def remove(storage, key)
60
+ begin
61
+ storage.where(key: key).delete
62
+ rescue
63
+ end
64
+ end
65
+
66
+ def clear_all(storage)
67
+ storage.delete_all
68
+ end
69
+ end
70
+ end
71
+
72
+ def get_node(node_id); Proxy.get(DCellNode, node_id) end
73
+ def set_node(node_id, addr); Proxy.set(DCellNode, node_id, addr) end
74
+ def nodes; Proxy.all(DCellNode) end
75
+ def remove_node(node_id); Proxy.remove(DCellNode, node_id) end
76
+ def clear_all_nodes; Proxy.clear_all(DCellNode) end
77
+
78
+ def get_global(key); Proxy.get(DCellGlobal, key) end
79
+ def set_global(key, value); Proxy.set(DCellGlobal, key, value) end
80
+ def global_keys; Proxy.all(DCellGlobal) end
81
+ def clear_globals; Proxy.clear_all(DCellGlobal) end
82
+
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,95 @@
1
+ require 'celluloid/redis'
2
+ require 'redis/connection/celluloid'
3
+ require 'redis-namespace'
4
+
5
+ module DCell
6
+ module Registry
7
+ class RedisAdapter
8
+ def initialize(options)
9
+ # Convert all options to symbols :/
10
+ options = options.inject({}) { |h,(k,v)| h[k.to_sym] = v; h }
11
+
12
+ @env = options[:env] || 'production'
13
+ @namespace = options[:namespace] || "dcell_#{@env}"
14
+
15
+ redis = Redis.new options
16
+ @redis = Redis::Namespace.new @namespace, :redis => redis
17
+
18
+ @node_registry = NodeRegistry.new(@redis)
19
+ @global_registry = GlobalRegistry.new(@redis)
20
+ end
21
+
22
+ def remove_node(node)
23
+ @node_registry.remove node
24
+ end
25
+
26
+ def clear_all_nodes
27
+ @node_registry.clear_all
28
+ end
29
+
30
+ def clear_globals
31
+ @global_registry.clear_all
32
+ end
33
+
34
+ class NodeRegistry
35
+ def initialize(redis)
36
+ @redis = redis
37
+ end
38
+
39
+ def get(node_id)
40
+ @redis.hget 'nodes', node_id
41
+ end
42
+
43
+ def set(node_id, addr)
44
+ @redis.hset 'nodes', node_id, addr
45
+ end
46
+
47
+ def nodes
48
+ @redis.hkeys 'nodes'
49
+ end
50
+
51
+ def remove(node)
52
+ @redis.hdel 'nodes', node
53
+ end
54
+
55
+ def clear_all
56
+ @redis.del 'nodes'
57
+ end
58
+ end
59
+
60
+ def get_node(node_id); @node_registry.get(node_id) end
61
+ def set_node(node_id, addr); @node_registry.set(node_id, addr) end
62
+ def nodes; @node_registry.nodes end
63
+
64
+ class GlobalRegistry
65
+ def initialize(redis)
66
+ @redis = redis
67
+ end
68
+
69
+ def get(key)
70
+ string = @redis.hget 'globals', key.to_s
71
+ Marshal.load string if string
72
+ end
73
+
74
+ # Set a global value
75
+ def set(key, value)
76
+ string = Marshal.dump value
77
+ @redis.hset 'globals', key.to_s, string
78
+ end
79
+
80
+ # The keys to all globals in the system
81
+ def global_keys
82
+ @redis.hkeys 'globals'
83
+ end
84
+
85
+ def clear_all
86
+ @redis.del 'globals'
87
+ end
88
+ end
89
+
90
+ def get_global(key); @global_registry.get(key) end
91
+ def set_global(key, value); @global_registry.set(key, value) end
92
+ def global_keys; @global_registry.global_keys end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,123 @@
1
+ require 'zk'
2
+
3
+ module DCell
4
+ module Registry
5
+ class ZkAdapter
6
+ PREFIX = "/dcell"
7
+ DEFAULT_PORT = 2181
8
+
9
+ # Create a new connection to Zookeeper
10
+ #
11
+ # servers: a list of Zookeeper servers to connect to. Each server in the
12
+ # list has a host/port configuration
13
+ def initialize(options)
14
+ # Stringify keys :/
15
+ options = options.inject({}) { |h,(k,v)| h[k.to_s] = v; h }
16
+
17
+ @env = options['env'] || 'production'
18
+ @base_path = "#{PREFIX}/#{@env}"
19
+
20
+ # Let them specify a single server instead of many
21
+ server = options['server']
22
+ if server
23
+ servers = [server]
24
+ else
25
+ servers = options['servers']
26
+ raise "no Zookeeper servers given" unless servers
27
+ end
28
+
29
+ # Add the default Zookeeper port unless specified
30
+ servers.map! do |server|
31
+ if server[/:\d+$/]
32
+ server
33
+ else
34
+ "#{server}:#{DEFAULT_PORT}"
35
+ end
36
+ end
37
+
38
+ @zk = ZK.new(*servers)
39
+ @node_registry = Registry.new(@zk, @base_path, :nodes, true)
40
+ @global_registry = Registry.new(@zk, @base_path, :globals, false)
41
+ end
42
+
43
+ class Registry
44
+ def initialize(zk, base_path, name, ephemeral)
45
+ @zk = zk
46
+ @base_path = File.join(base_path, name.to_s)
47
+ @ephemeral = ephemeral
48
+ @zk.mkdir_p @base_path
49
+ @events = {}
50
+ end
51
+
52
+ def register(key)
53
+ path = "#{@base_path}/#{key}"
54
+ @events[path] ||= @zk.register(path) do |event|
55
+ key = event.path.match(/#{@base_path}\/(\w+)/)[1]
56
+ @events[event.path].unsubscribe
57
+ @events[event.path] = nil
58
+ if event.node_changed?
59
+ Celluloid::logger.debug "zk callback: node changed!"
60
+ Node.update(key)
61
+ end
62
+ if event.node_deleted?
63
+ Celluloid::logger.debug "zk callback: node deleted!"
64
+ Node.remove(key)
65
+ end
66
+
67
+ end
68
+ end
69
+
70
+ def get(key)
71
+ register(key)
72
+ result, _ = @zk.get("#{@base_path}/#{key}", watch: true)
73
+ Marshal.load result
74
+ rescue ZK::Exceptions::NoNode
75
+ end
76
+
77
+ def set(key, value)
78
+ register(key)
79
+ path = "#{@base_path}/#{key}"
80
+ string = Marshal.dump value
81
+ @zk.set path, string
82
+ rescue ZK::Exceptions::NoNode
83
+ @zk.create path, string, :ephemeral => @ephemeral
84
+ end
85
+
86
+ def all
87
+ @zk.children @base_path
88
+ end
89
+
90
+ def remove(key)
91
+ keys.each do |key|
92
+ path = "#{@base_path}/#{key}"
93
+ @zk.delete path
94
+ end
95
+ end
96
+
97
+ def clear_all
98
+ # delete znodes so any registered
99
+ # callback is triggered
100
+ all.each do |key|
101
+ remove key
102
+ end
103
+ @events.values.compact.each(&:unsubscribe)
104
+ @events.clear
105
+ @zk.rm_rf @base_path
106
+ @zk.mkdir_p @base_path
107
+ end
108
+ end
109
+
110
+ def get_node(node_id); @node_registry.get(node_id) end
111
+ def set_node(node_id, addr); @node_registry.set(node_id, addr) end
112
+ def nodes; @node_registry.all end
113
+ def remove_node(node_id); @node_registry.clear(node_id) end
114
+ def clear_all_nodes; @node_registry.clear_all end
115
+
116
+ def get_global(key); @global_registry.get(key) end
117
+ def set_global(key, value); @global_registry.set(key, value) end
118
+ def global_keys; @global_registry.all end
119
+ def clear_globals; @global_registry.clear_all end
120
+
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,16 @@
1
+ module DCell
2
+ # Responses to calls
3
+ class Response
4
+ attr_reader :request_id, :value
5
+
6
+ def initialize(request_id, value)
7
+ @request_id, @value = request_id, value
8
+ end
9
+ end
10
+
11
+ # Request successful
12
+ class SuccessResponse < Response; end
13
+
14
+ # Request failed
15
+ class ErrorResponse < Response; end
16
+ end