stn-dcell 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +1 -0
  3. data/.gitignore +7 -0
  4. data/.rspec +4 -0
  5. data/.travis.yml +30 -0
  6. data/CHANGES.md +53 -0
  7. data/Gemfile +18 -0
  8. data/LICENSE.txt +20 -0
  9. data/README.md +168 -0
  10. data/Rakefile +4 -0
  11. data/benchmarks/messaging.rb +73 -0
  12. data/benchmarks/receiver.rb +37 -0
  13. data/dcell.gemspec +29 -0
  14. data/examples/itchy.rb +26 -0
  15. data/examples/scratchy.rb +12 -0
  16. data/explorer/css/bootstrap-responsive.css +686 -0
  17. data/explorer/css/bootstrap-responsive.min.css +12 -0
  18. data/explorer/css/bootstrap.css +3990 -0
  19. data/explorer/css/bootstrap.min.css +689 -0
  20. data/explorer/css/explorer.css +28 -0
  21. data/explorer/ico/favicon.ico +0 -0
  22. data/explorer/img/glyphicons-halflings-white.png +0 -0
  23. data/explorer/img/glyphicons-halflings.png +0 -0
  24. data/explorer/img/logo.png +0 -0
  25. data/explorer/index.html.erb +94 -0
  26. data/explorer/js/bootstrap.js +1726 -0
  27. data/explorer/js/bootstrap.min.js +6 -0
  28. data/lib/dcell.rb +127 -0
  29. data/lib/dcell/actor_proxy.rb +30 -0
  30. data/lib/dcell/celluloid_ext.rb +120 -0
  31. data/lib/dcell/directory.rb +31 -0
  32. data/lib/dcell/explorer.rb +74 -0
  33. data/lib/dcell/future_proxy.rb +32 -0
  34. data/lib/dcell/global.rb +23 -0
  35. data/lib/dcell/info_service.rb +122 -0
  36. data/lib/dcell/mailbox_proxy.rb +53 -0
  37. data/lib/dcell/messages.rb +65 -0
  38. data/lib/dcell/node.rb +138 -0
  39. data/lib/dcell/node_manager.rb +79 -0
  40. data/lib/dcell/registries/cassandra_adapter.rb +126 -0
  41. data/lib/dcell/registries/mongodb_adapter.rb +85 -0
  42. data/lib/dcell/registries/redis_adapter.rb +95 -0
  43. data/lib/dcell/registries/zk_adapter.rb +123 -0
  44. data/lib/dcell/responses.rb +16 -0
  45. data/lib/dcell/router.rb +46 -0
  46. data/lib/dcell/rpc.rb +95 -0
  47. data/lib/dcell/rspec.rb +1 -0
  48. data/lib/dcell/server.rb +73 -0
  49. data/lib/dcell/version.rb +3 -0
  50. data/logo.png +0 -0
  51. data/spec/dcell/actor_proxy_spec.rb +72 -0
  52. data/spec/dcell/celluloid_ext_spec.rb +32 -0
  53. data/spec/dcell/directory_spec.rb +22 -0
  54. data/spec/dcell/explorer_spec.rb +17 -0
  55. data/spec/dcell/global_spec.rb +25 -0
  56. data/spec/dcell/node_spec.rb +23 -0
  57. data/spec/dcell/registries/mongodb_adapter_spec.rb +7 -0
  58. data/spec/dcell/registries/redis_adapter_spec.rb +6 -0
  59. data/spec/dcell/registries/zk_adapter_spec.rb +28 -0
  60. data/spec/options/01-options.rb +10 -0
  61. data/spec/options/02-registry.rb +13 -0
  62. data/spec/spec_helper.rb +28 -0
  63. data/spec/support/helpers.rb +46 -0
  64. data/spec/support/registry_examples.rb +35 -0
  65. data/spec/test_node.rb +38 -0
  66. data/tasks/cassandra.task +84 -0
  67. data/tasks/rspec.task +7 -0
  68. data/tasks/zookeeper.task +58 -0
  69. metadata +239 -0
@@ -0,0 +1,126 @@
1
+ require 'cassandra'
2
+
3
+ # create the keyspace / columnfamily with cqlsh
4
+ #
5
+ # create keyspace dcell
6
+ # with strategy_class='SimpleStrategy'
7
+ # and strategy_options:replication_factor=3;
8
+ #
9
+ # create columnfamily dcell (dcell_type ascii primary key);
10
+
11
+ # not sure this is right yet ...
12
+ # Keyspace "whatever" [
13
+ # ColumnFamily "dcell" {
14
+ # RowKey "nodes": {
15
+ # <nodeid>: <address>,
16
+ # <nodeid>: <address>,
17
+ # ...
18
+ # }
19
+ # RowKey "globals": {
20
+ # <key>: <marshal blob>,
21
+ # <key>: <marshal blob>,
22
+ # ...
23
+ # }
24
+ # }
25
+ # ]
26
+ #
27
+
28
+ module DCell
29
+ module Registry
30
+ class CassandraAdapter
31
+ DEFAULT_KEYSPACE = "dcell"
32
+ DEFAULT_CF = "dcell"
33
+
34
+ def initialize(options)
35
+ # Convert all options to symbols :/
36
+ options = options.inject({}) { |h,(k,v)| h[k.to_sym] = v; h }
37
+
38
+ keyspace = options[:keyspace] || DEFAULT_KEYSPACE
39
+ columnfamily = options[:columnfamily] || DEFAULT_CF
40
+
41
+ options[:servers] ||= []
42
+ options[:servers] << options[:server] if options[:server]
43
+ options[:servers] << "localhost:9160" unless options[:servers].any?
44
+
45
+ cass = Cassandra.new(keyspace, options[:servers])
46
+
47
+ @node_registry = NodeRegistry.new(cass, columnfamily)
48
+ @global_registry = GlobalRegistry.new(cass, columnfamily)
49
+ end
50
+
51
+ def remove_node(node)
52
+ @node_registry.remove node
53
+ end
54
+
55
+ def clear_all_nodes
56
+ @node_registry.clear_all
57
+ end
58
+
59
+ def clear_globals
60
+ @global_registry.clear_all
61
+ end
62
+
63
+ class NodeRegistry
64
+ def initialize(cass, cf)
65
+ @cass = cass
66
+ @cf = cf
67
+ end
68
+
69
+ def get(node_id)
70
+ @cass.get @cf, "nodes", node_id
71
+ end
72
+
73
+ def set(node_id, addr)
74
+ @cass.insert @cf, "nodes", { node_id => addr }
75
+ end
76
+
77
+ def nodes
78
+ @cass.get(@cf, "nodes").keys
79
+ end
80
+
81
+ def remove(node)
82
+ @cass.remove @cf, "nodes", { node_id => node }
83
+ end
84
+
85
+ def clear_all
86
+ @cass.del @cf, "nodes"
87
+ end
88
+ end
89
+
90
+ def get_node(node_id); @node_registry.get(node_id) end
91
+ def set_node(node_id, addr); @node_registry.set(node_id, addr) end
92
+ def nodes; @node_registry.nodes end
93
+
94
+ class GlobalRegistry
95
+ def initialize(cass, cf)
96
+ @cass = cass
97
+ @cf = cf
98
+ end
99
+
100
+ def get(key)
101
+ string = @cass.get @cf, "globals", key.to_s
102
+ Marshal.load string if string
103
+ end
104
+
105
+ # Set a global value
106
+ def set(key, value)
107
+ string = Marshal.dump value
108
+ @cass.insert @cf, "globals", { key.to_s => string }
109
+ end
110
+
111
+ # The keys to all globals in the system
112
+ def global_keys
113
+ @cass.get(@cf, "globals").keys
114
+ end
115
+
116
+ def clear_all
117
+ @cass.del @cf, "globals"
118
+ end
119
+ end
120
+
121
+ def get_global(key); @global_registry.get(key) end
122
+ def set_global(key, value); @global_registry.set(key, value) end
123
+ def global_keys; @global_registry.global_keys end
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,85 @@
1
+ require 'mongoid'
2
+
3
+ module DCell
4
+ module Registry
5
+ class MongodbAdapter
6
+ # Setup connection to mongodb
7
+ # config: path to mongoid configuration file
8
+ # env: mongoid environment to use
9
+ def initialize(options)
10
+ if options[:config]
11
+ Mongoid.load! options[:config], options[:env]
12
+ elsif options[:db]
13
+ Mongoid.connect_to options[:db]
14
+ end
15
+ if options[:options]
16
+ Mongoid.options = options[:options]
17
+ end
18
+ end
19
+
20
+ class DCellNode
21
+ include Mongoid::Document
22
+
23
+ field :key, type: String
24
+ field :value, type: Hash
25
+ end
26
+
27
+ class DCellGlobal
28
+ include Mongoid::Document
29
+
30
+ field :key, type: String
31
+ field :value, type: Hash
32
+ end
33
+
34
+ class Proxy
35
+ class << self
36
+ def set(storage, key, value)
37
+ entry = storage.find_or_create_by(key: key)
38
+ entry.value = {'v' => value}
39
+ entry.save!
40
+ value
41
+ end
42
+
43
+ def get(storage, key)
44
+ first = storage.where(key: key).first
45
+ if first and first.value
46
+ return first.value['v']
47
+ end
48
+ nil
49
+ end
50
+
51
+ def all(storage)
52
+ keys = []
53
+ storage.each do |entry|
54
+ keys << entry.key
55
+ end
56
+ keys
57
+ end
58
+
59
+ def remove(storage, key)
60
+ begin
61
+ storage.where(key: key).delete
62
+ rescue
63
+ end
64
+ end
65
+
66
+ def clear_all(storage)
67
+ storage.delete_all
68
+ end
69
+ end
70
+ end
71
+
72
+ def get_node(node_id); Proxy.get(DCellNode, node_id) end
73
+ def set_node(node_id, addr); Proxy.set(DCellNode, node_id, addr) end
74
+ def nodes; Proxy.all(DCellNode) end
75
+ def remove_node(node_id); Proxy.remove(DCellNode, node_id) end
76
+ def clear_all_nodes; Proxy.clear_all(DCellNode) end
77
+
78
+ def get_global(key); Proxy.get(DCellGlobal, key) end
79
+ def set_global(key, value); Proxy.set(DCellGlobal, key, value) end
80
+ def global_keys; Proxy.all(DCellGlobal) end
81
+ def clear_globals; Proxy.clear_all(DCellGlobal) end
82
+
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,95 @@
1
+ require 'celluloid/redis'
2
+ require 'redis/connection/celluloid'
3
+ require 'redis-namespace'
4
+
5
+ module DCell
6
+ module Registry
7
+ class RedisAdapter
8
+ def initialize(options)
9
+ # Convert all options to symbols :/
10
+ options = options.inject({}) { |h,(k,v)| h[k.to_sym] = v; h }
11
+
12
+ @env = options[:env] || 'production'
13
+ @namespace = options[:namespace] || "dcell_#{@env}"
14
+
15
+ redis = Redis.new options
16
+ @redis = Redis::Namespace.new @namespace, :redis => redis
17
+
18
+ @node_registry = NodeRegistry.new(@redis)
19
+ @global_registry = GlobalRegistry.new(@redis)
20
+ end
21
+
22
+ def remove_node(node)
23
+ @node_registry.remove node
24
+ end
25
+
26
+ def clear_all_nodes
27
+ @node_registry.clear_all
28
+ end
29
+
30
+ def clear_globals
31
+ @global_registry.clear_all
32
+ end
33
+
34
+ class NodeRegistry
35
+ def initialize(redis)
36
+ @redis = redis
37
+ end
38
+
39
+ def get(node_id)
40
+ @redis.hget 'nodes', node_id
41
+ end
42
+
43
+ def set(node_id, addr)
44
+ @redis.hset 'nodes', node_id, addr
45
+ end
46
+
47
+ def nodes
48
+ @redis.hkeys 'nodes'
49
+ end
50
+
51
+ def remove(node)
52
+ @redis.hdel 'nodes', node
53
+ end
54
+
55
+ def clear_all
56
+ @redis.del 'nodes'
57
+ end
58
+ end
59
+
60
+ def get_node(node_id); @node_registry.get(node_id) end
61
+ def set_node(node_id, addr); @node_registry.set(node_id, addr) end
62
+ def nodes; @node_registry.nodes end
63
+
64
+ class GlobalRegistry
65
+ def initialize(redis)
66
+ @redis = redis
67
+ end
68
+
69
+ def get(key)
70
+ string = @redis.hget 'globals', key.to_s
71
+ Marshal.load string if string
72
+ end
73
+
74
+ # Set a global value
75
+ def set(key, value)
76
+ string = Marshal.dump value
77
+ @redis.hset 'globals', key.to_s, string
78
+ end
79
+
80
+ # The keys to all globals in the system
81
+ def global_keys
82
+ @redis.hkeys 'globals'
83
+ end
84
+
85
+ def clear_all
86
+ @redis.del 'globals'
87
+ end
88
+ end
89
+
90
+ def get_global(key); @global_registry.get(key) end
91
+ def set_global(key, value); @global_registry.set(key, value) end
92
+ def global_keys; @global_registry.global_keys end
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,123 @@
1
+ require 'zk'
2
+
3
+ module DCell
4
+ module Registry
5
+ class ZkAdapter
6
+ PREFIX = "/dcell"
7
+ DEFAULT_PORT = 2181
8
+
9
+ # Create a new connection to Zookeeper
10
+ #
11
+ # servers: a list of Zookeeper servers to connect to. Each server in the
12
+ # list has a host/port configuration
13
+ def initialize(options)
14
+ # Stringify keys :/
15
+ options = options.inject({}) { |h,(k,v)| h[k.to_s] = v; h }
16
+
17
+ @env = options['env'] || 'production'
18
+ @base_path = "#{PREFIX}/#{@env}"
19
+
20
+ # Let them specify a single server instead of many
21
+ server = options['server']
22
+ if server
23
+ servers = [server]
24
+ else
25
+ servers = options['servers']
26
+ raise "no Zookeeper servers given" unless servers
27
+ end
28
+
29
+ # Add the default Zookeeper port unless specified
30
+ servers.map! do |server|
31
+ if server[/:\d+$/]
32
+ server
33
+ else
34
+ "#{server}:#{DEFAULT_PORT}"
35
+ end
36
+ end
37
+
38
+ @zk = ZK.new(*servers)
39
+ @node_registry = Registry.new(@zk, @base_path, :nodes, true)
40
+ @global_registry = Registry.new(@zk, @base_path, :globals, false)
41
+ end
42
+
43
+ class Registry
44
+ def initialize(zk, base_path, name, ephemeral)
45
+ @zk = zk
46
+ @base_path = File.join(base_path, name.to_s)
47
+ @ephemeral = ephemeral
48
+ @zk.mkdir_p @base_path
49
+ @events = {}
50
+ end
51
+
52
+ def register(key)
53
+ path = "#{@base_path}/#{key}"
54
+ @events[path] ||= @zk.register(path) do |event|
55
+ key = event.path.match(/#{@base_path}\/(\w+)/)[1]
56
+ @events[event.path].unsubscribe
57
+ @events[event.path] = nil
58
+ if event.node_changed?
59
+ Celluloid::logger.debug "zk callback: node changed!"
60
+ Node.update(key)
61
+ end
62
+ if event.node_deleted?
63
+ Celluloid::logger.debug "zk callback: node deleted!"
64
+ Node.remove(key)
65
+ end
66
+
67
+ end
68
+ end
69
+
70
+ def get(key)
71
+ register(key)
72
+ result, _ = @zk.get("#{@base_path}/#{key}", watch: true)
73
+ Marshal.load result
74
+ rescue ZK::Exceptions::NoNode
75
+ end
76
+
77
+ def set(key, value)
78
+ register(key)
79
+ path = "#{@base_path}/#{key}"
80
+ string = Marshal.dump value
81
+ @zk.set path, string
82
+ rescue ZK::Exceptions::NoNode
83
+ @zk.create path, string, :ephemeral => @ephemeral
84
+ end
85
+
86
+ def all
87
+ @zk.children @base_path
88
+ end
89
+
90
+ def remove(key)
91
+ keys.each do |key|
92
+ path = "#{@base_path}/#{key}"
93
+ @zk.delete path
94
+ end
95
+ end
96
+
97
+ def clear_all
98
+ # delete znodes so any registered
99
+ # callback is triggered
100
+ all.each do |key|
101
+ remove key
102
+ end
103
+ @events.values.compact.each(&:unsubscribe)
104
+ @events.clear
105
+ @zk.rm_rf @base_path
106
+ @zk.mkdir_p @base_path
107
+ end
108
+ end
109
+
110
+ def get_node(node_id); @node_registry.get(node_id) end
111
+ def set_node(node_id, addr); @node_registry.set(node_id, addr) end
112
+ def nodes; @node_registry.all end
113
+ def remove_node(node_id); @node_registry.clear(node_id) end
114
+ def clear_all_nodes; @node_registry.clear_all end
115
+
116
+ def get_global(key); @global_registry.get(key) end
117
+ def set_global(key, value); @global_registry.set(key, value) end
118
+ def global_keys; @global_registry.all end
119
+ def clear_globals; @global_registry.clear_all end
120
+
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,16 @@
1
+ module DCell
2
+ # Responses to calls
3
+ class Response
4
+ attr_reader :request_id, :value
5
+
6
+ def initialize(request_id, value)
7
+ @request_id, @value = request_id, value
8
+ end
9
+ end
10
+
11
+ # Request successful
12
+ class SuccessResponse < Response; end
13
+
14
+ # Request failed
15
+ class ErrorResponse < Response; end
16
+ end