ruby_skynet 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +11 -19
- data/lib/ruby_skynet/railtie.rb +2 -2
- data/lib/ruby_skynet/railties/ruby_skynet.rake +5 -5
- data/lib/ruby_skynet/ruby_skynet.rb +2 -2
- data/lib/ruby_skynet/version.rb +1 -1
- data/lib/ruby_skynet/zookeeper/registry.rb +100 -70
- data/lib/ruby_skynet/zookeeper/service_registry.rb +23 -11
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: db463c6efbc6b84b670681ac67fd39358dcdcfd1
|
4
|
+
data.tar.gz: 6f18c09efa035cc23435b5a29a8a16c75dcd7437
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e763b084264b1073831234160c328f9e594d0d33c228240b7487bac01c5fcb9422fd20910a0e3beeca0a38d5cb5f7e1c8521bb49c7751bdbc790bc974c815cb8
|
7
|
+
data.tar.gz: 41976bc92888af6ba2d1e20d75b5babac0d1b5b03b0f38da10489d2e4e9121557bca532c1e9f1ca1f76e8c2adbcc05e6c1af331ae77edd5d0782304bde56bf5f
|
data/Gemfile.lock
CHANGED
@@ -7,20 +7,18 @@ GEM
|
|
7
7
|
multi_json (~> 1.3)
|
8
8
|
thread_safe (~> 0.1)
|
9
9
|
tzinfo (~> 0.3.37)
|
10
|
-
atomic (1.1.
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
bson_ext (1.9.0)
|
15
|
-
bson (~> 1.9.0)
|
10
|
+
atomic (1.1.13)
|
11
|
+
bson (1.9.1)
|
12
|
+
bson_ext (1.9.1)
|
13
|
+
bson (~> 1.9.1)
|
16
14
|
gene_pool (1.3.0)
|
17
|
-
i18n (0.6.
|
15
|
+
i18n (0.6.5)
|
18
16
|
minitest (4.7.5)
|
19
|
-
multi_json (1.7.
|
17
|
+
multi_json (1.7.9)
|
20
18
|
rake (10.1.0)
|
21
19
|
resilient_socket (0.5.0)
|
22
20
|
semantic_logger (>= 2.1)
|
23
|
-
ruby_doozer (0.
|
21
|
+
ruby_doozer (0.8.1)
|
24
22
|
gene_pool (>= 1.3.0)
|
25
23
|
multi_json (>= 1.6.1)
|
26
24
|
resilient_socket (>= 0.5.0)
|
@@ -28,28 +26,22 @@ GEM
|
|
28
26
|
semantic_logger (>= 2.1)
|
29
27
|
sync_attr (>= 1.0.0)
|
30
28
|
ruby_protobuf (0.4.11)
|
31
|
-
semantic_logger (2.
|
29
|
+
semantic_logger (2.2.0)
|
32
30
|
sync_attr (>= 1.0)
|
33
31
|
thread_safe (>= 0.1.0)
|
34
32
|
shoulda (3.5.0)
|
35
33
|
shoulda-context (~> 1.0, >= 1.0.1)
|
36
34
|
shoulda-matchers (>= 1.4.1, < 3.0)
|
37
|
-
shoulda-context (1.1.
|
38
|
-
shoulda-matchers (2.
|
35
|
+
shoulda-context (1.1.5)
|
36
|
+
shoulda-matchers (2.3.0)
|
39
37
|
activesupport (>= 3.0.0)
|
40
|
-
slyphon-log4j (1.2.15)
|
41
|
-
slyphon-zookeeper_jar (3.3.5-java)
|
42
38
|
sync_attr (1.0.0)
|
43
|
-
thread_safe (0.1.
|
39
|
+
thread_safe (0.1.2)
|
44
40
|
atomic
|
45
41
|
tzinfo (0.3.37)
|
46
42
|
zookeeper (1.4.4)
|
47
|
-
zookeeper (1.4.4-java)
|
48
|
-
slyphon-log4j (= 1.2.15)
|
49
|
-
slyphon-zookeeper_jar (= 3.3.5)
|
50
43
|
|
51
44
|
PLATFORMS
|
52
|
-
java
|
53
45
|
ruby
|
54
46
|
|
55
47
|
DEPENDENCIES
|
data/lib/ruby_skynet/railtie.rb
CHANGED
@@ -15,8 +15,8 @@ module RubySkynet #:nodoc:
|
|
15
15
|
load "ruby_skynet/railties/ruby_skynet.rake"
|
16
16
|
end
|
17
17
|
|
18
|
-
# Load RubySkynet Configuration
|
19
|
-
|
18
|
+
# Load RubySkynet Configuration once rails has started
|
19
|
+
initializer 'ruby_skynet.initialize' do
|
20
20
|
config_file = Rails.root.join("config", "ruby_skynet.yml")
|
21
21
|
if config_file.file?
|
22
22
|
::RubySkynet.configure!(config_file, Rails.env)
|
@@ -20,14 +20,14 @@ namespace :ruby_skynet do
|
|
20
20
|
|
21
21
|
RubySkynet::Server.load_services
|
22
22
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
begin
|
24
|
+
# Start the server
|
25
|
+
RubySkynet::Server.start
|
26
|
+
RubySkynet::Server.wait_until_server_stops
|
27
|
+
ensure
|
27
28
|
RubySkynet::Server.stop
|
28
29
|
end
|
29
30
|
|
30
|
-
RubySkynet::Server.wait_until_server_stops
|
31
31
|
end
|
32
32
|
|
33
33
|
end
|
@@ -98,10 +98,10 @@ module RubySkynet
|
|
98
98
|
|
99
99
|
# Extract just the zookeeper or doozer configuration element
|
100
100
|
RubySkynet.service_registry = ServiceRegistry.new(
|
101
|
-
:registry => config
|
101
|
+
:registry => config.delete(:registry)
|
102
102
|
)
|
103
103
|
|
104
|
-
config.each_pair {|k,v|
|
104
|
+
config.each_pair {|k,v| warn "Ignoring unknown RubySkynet config option #{k} => #{v}"}
|
105
105
|
end
|
106
106
|
|
107
107
|
# Initialize internal class variable
|
data/lib/ruby_skynet/version.rb
CHANGED
@@ -37,6 +37,10 @@ module RubySkynet
|
|
37
37
|
# :ephemeral [Boolean]
|
38
38
|
# All set operations of non-nil values will result in ephemeral nodes.
|
39
39
|
#
|
40
|
+
# :on_connect [Proc]
|
41
|
+
# Block to call after the connection to Zookeeper has been established
|
42
|
+
# and every time the connection is re-established
|
43
|
+
#
|
40
44
|
# :registry [Hash|ZooKeeper]
|
41
45
|
# ZooKeeper configuration information, or an existing
|
42
46
|
# ZooKeeper ( ZooKeeper client) instance
|
@@ -72,19 +76,13 @@ module RubySkynet
|
|
72
76
|
@root = '/' if @root == ''
|
73
77
|
|
74
78
|
registry_config = params.delete(:registry) || {}
|
75
|
-
if registry_config.is_a?(::Zookeeper::Client)
|
76
|
-
@zookeeper = registry_config
|
77
|
-
else
|
78
|
-
servers = registry_config.delete(:servers) || ['127.0.0.1:2181']
|
79
|
-
connect_timeout = (registry_config.delete(:connect_timeout) || 10).to_f
|
80
79
|
|
81
|
-
|
82
|
-
|
80
|
+
# server1:2181,server2:2181,server3:2181
|
81
|
+
@servers = (registry_config.delete(:servers) || ['127.0.0.1:2181']).join(',')
|
82
|
+
@connect_timeout = (registry_config.delete(:connect_timeout) || 10).to_f
|
83
83
|
|
84
|
-
|
85
|
-
|
86
|
-
@zookeeper = ::Zookeeper.new(servers.join(','), connect_timeout, watcher)
|
87
|
-
end
|
84
|
+
# Generate warning log entries for any unknown configuration options
|
85
|
+
registry_config.each_pair {|k,v| logger.warn "Ignoring unknown configuration option: zookeeper.#{k}"}
|
88
86
|
|
89
87
|
# Allow the serializer and deserializer implementations to be replaced
|
90
88
|
@serializer = params.delete(:serializer) || RubySkynet::Zookeeper::Json::Serializer
|
@@ -93,18 +91,18 @@ module RubySkynet
|
|
93
91
|
@ephemeral = params.delete(:ephemeral)
|
94
92
|
@ephemeral = false if @ephemeral.nil?
|
95
93
|
|
94
|
+
@on_connect = params.delete(:on_connect)
|
95
|
+
|
96
96
|
# Generate warning log entries for any unknown configuration options
|
97
97
|
params.each_pair {|k,v| logger.warn "Ignoring unknown configuration option: #{k}"}
|
98
98
|
|
99
99
|
# Hash with Array values containing the list of children for each node, if any
|
100
100
|
@children = ThreadSafe::Hash.new
|
101
101
|
|
102
|
-
#
|
103
|
-
|
102
|
+
# Block is used in init
|
103
|
+
@block = block
|
104
104
|
|
105
|
-
|
106
|
-
close
|
107
|
-
end
|
105
|
+
self.init
|
108
106
|
end
|
109
107
|
|
110
108
|
# Retrieve the latest value from a specific path from the registry
|
@@ -326,69 +324,83 @@ module RubySkynet
|
|
326
324
|
end
|
327
325
|
end
|
328
326
|
|
329
|
-
# returns the watcher proc for this registry
|
327
|
+
# returns the watcher proc for this registry instance
|
330
328
|
def watcher
|
331
329
|
# Subscription block to call for watch events
|
332
330
|
@watch_proc ||= Proc.new do |event_hash|
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
331
|
+
begin
|
332
|
+
path = event_hash[:path]
|
333
|
+
logger.trace "Event Received", event_hash
|
334
|
+
case event_hash[:type]
|
335
|
+
when ::Zookeeper::ZOO_CHANGED_EVENT
|
336
|
+
logger.debug "Node '#{path}' Changed", event_hash
|
337
|
+
|
338
|
+
# Fetch current value and re-subscribe
|
339
|
+
result = @zookeeper.get(:path => path, :watcher => @watch_proc)
|
340
|
+
check_rc(result)
|
341
|
+
value = @deserializer.deserialize(result[:data])
|
342
|
+
stat = result[:stat]
|
343
|
+
|
344
|
+
# Invoke on_update callbacks
|
345
|
+
node_updated(relative_key(path), value, stat.version)
|
346
|
+
|
347
|
+
when ::Zookeeper::ZOO_DELETED_EVENT
|
348
|
+
# A node has been deleted
|
349
|
+
# TODO How to ignore child deleted when it is a directory, not a leaf
|
350
|
+
logger.debug "Node '#{path}' Deleted", event_hash
|
351
|
+
@children.delete(path)
|
352
|
+
node_deleted(relative_key(path))
|
353
|
+
|
354
|
+
when ::Zookeeper::ZOO_CHILD_EVENT
|
355
|
+
# The list of nodes has changed - Does not say if it was added or removed
|
356
|
+
logger.debug "Node '#{path}' Child changed", event_hash
|
357
|
+
result = @zookeeper.get_children(:path => path, :watcher => @watch_proc)
|
358
|
+
|
359
|
+
# This node could have been deleted already
|
360
|
+
if result[:rc] == ::Zookeeper::ZOK
|
361
|
+
current_children = result[:children]
|
362
|
+
previous_children = @children[path]
|
363
|
+
|
364
|
+
# Save children so that we can later identify new children
|
365
|
+
@children[path] = current_children
|
366
|
+
|
367
|
+
# New Child Nodes
|
368
|
+
new_nodes = previous_children ? (current_children - previous_children) : current_children
|
369
|
+
new_nodes.each do |child|
|
370
|
+
get_recursive(File.join(path,child), true) do |key, value, version|
|
371
|
+
node_created(key, value, version)
|
372
|
+
end
|
372
373
|
end
|
374
|
+
# Ignore Deleted Child Nodes since they will be handled by the Deleted Node event
|
373
375
|
end
|
374
|
-
# Ignore Deleted Child Nodes since they will be handled by the Deleted Node event
|
375
|
-
end
|
376
376
|
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
377
|
+
when ::Zookeeper::ZOO_CREATED_EVENT
|
378
|
+
# Node created events are only created for paths that were deleted
|
379
|
+
# and then created again
|
380
|
+
# No op - This is covered by node_child created event
|
381
|
+
logger.debug "Node '#{path}' Created - No op", event_hash
|
382
|
+
|
383
|
+
when ::Zookeeper::ZOO_SESSION_EVENT
|
384
|
+
logger.debug "Session Event: #{@zookeeper.state_by_value(event_hash[:state]) if @zookeeper}", event_hash
|
382
385
|
|
383
|
-
|
384
|
-
|
386
|
+
# Replace zookeeper connection since it is stale. Only react to global request
|
387
|
+
# since this event will be received for every node being watched.
|
388
|
+
# Do not close the current connection since this background watcher thread is running
|
389
|
+
# as part of the current zookeeper connection
|
390
|
+
# event_hash => {:req_id=>-1, :type=>-1, :state=>-112, :path=>"", :context=>nil}
|
391
|
+
Thread.new { self.init } if (event_hash[:req_id] == -1) && (event_hash[:state] == ::Zookeeper::ZOO_EXPIRED_SESSION_STATE)
|
385
392
|
|
386
|
-
|
387
|
-
|
393
|
+
when ::Zookeeper::ZOO_NOTWATCHING_EVENT
|
394
|
+
logger.debug "Ignoring ZOO_NOTWATCHING_EVENT", event_hash
|
388
395
|
|
389
|
-
|
390
|
-
|
391
|
-
|
396
|
+
else
|
397
|
+
# TODO Need to re-load registry when re-connected
|
398
|
+
logger.warn "Ignoring unknown event", event_hash
|
399
|
+
end
|
400
|
+
rescue ::Zookeeper::Exceptions::ZookeeperException => exc
|
401
|
+
logger.warn "Watching thread failed due to Zookeeper failure", exc
|
402
|
+
rescue Exception => exc
|
403
|
+
logger.error "Watching thread failed due to unhandled exception", exc
|
392
404
|
end
|
393
405
|
end
|
394
406
|
end
|
@@ -511,6 +523,24 @@ module RubySkynet
|
|
511
523
|
end
|
512
524
|
end
|
513
525
|
|
526
|
+
# Create ZooKeeper connection and start watching the registry for any changes
|
527
|
+
def init
|
528
|
+
logger.benchmark_info "Connected to Zookeeper" do
|
529
|
+
@zookeeper.close if @zookeeper
|
530
|
+
# Create Zookeeper connection
|
531
|
+
@zookeeper = ::Zookeeper.new(@servers, @connect_timeout, watcher)
|
532
|
+
at_exit do
|
533
|
+
@zookeeper.close if @zookeeper
|
534
|
+
end
|
535
|
+
|
536
|
+
# Start watching registry for any changes
|
537
|
+
get_recursive(@root, watch=true, create_path=true, &@block)
|
538
|
+
|
539
|
+
# Call on_connect callback if supplied
|
540
|
+
@on_connect.call(self) if @on_connect
|
541
|
+
end
|
542
|
+
end
|
543
|
+
|
514
544
|
end
|
515
545
|
end
|
516
546
|
end
|
@@ -23,9 +23,17 @@ module RubySkynet
|
|
23
23
|
@cache = ThreadSafe::Hash.new
|
24
24
|
@notifications_cache = ThreadSafe::Hash.new
|
25
25
|
|
26
|
+
# Keep a list of registered services so that they can be re-registered
|
27
|
+
# if the connection is lost
|
28
|
+
@services = ThreadSafe::Hash.new
|
29
|
+
|
26
30
|
# Supply block to load the current keys from the Registry
|
27
31
|
params[:root] = '/instances'
|
28
32
|
params[:ephemeral] = true
|
33
|
+
params[:on_connect] = Proc.new do |registry|
|
34
|
+
# Re-Register services every time the connection to ZooKeeper is lost
|
35
|
+
@services.values.each {|v| register_service(*v)}
|
36
|
+
end
|
29
37
|
@registry = Zookeeper::Registry.new(params) do |key, value|
|
30
38
|
service_info_created(key, value)
|
31
39
|
end
|
@@ -44,13 +52,14 @@ module RubySkynet
|
|
44
52
|
# Returns the UUID for the service that was created
|
45
53
|
def register_service(name, version, region, hostname, port)
|
46
54
|
uuid = "#{hostname}:#{port}-#{$$}-#{name}-#{version}"
|
47
|
-
# TODO Make sets ephemeral
|
48
55
|
@registry[File.join(uuid,'addr')] = "#{hostname}:#{port}"
|
49
56
|
@registry[File.join(uuid,'name')] = name
|
50
57
|
@registry[File.join(uuid,'version')] = version
|
51
58
|
@registry[File.join(uuid,'region')] = region
|
52
59
|
@registry[File.join(uuid,'registered')] = true
|
53
|
-
|
60
|
+
# Add to local services list
|
61
|
+
@services[uuid] = [name, version, region, hostname, port]
|
62
|
+
uuid
|
54
63
|
end
|
55
64
|
|
56
65
|
# Deregister the supplied service from the Registry
|
@@ -62,6 +71,9 @@ module RubySkynet
|
|
62
71
|
@registry.delete(File.join(uuid,'region'), false)
|
63
72
|
@registry.delete(File.join(uuid,'registered'), false)
|
64
73
|
@registry.delete(uuid, false)
|
74
|
+
# Remove from local services list
|
75
|
+
@services.delete(uuid)
|
76
|
+
uuid
|
65
77
|
end
|
66
78
|
|
67
79
|
# Return a server that implements the specified service
|
@@ -105,21 +117,21 @@ module RubySkynet
|
|
105
117
|
protected
|
106
118
|
|
107
119
|
def service_info_deleted(path)
|
108
|
-
logger.
|
120
|
+
logger.trace("service_info_deleted: #{path}")
|
109
121
|
# path: "uuid/key"
|
110
122
|
uuid, key = path.split('/')
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
123
|
+
# If any child of the node is deleted, deregister the service,
|
124
|
+
# not just for (key == 'registered')
|
125
|
+
if server = @notifications_cache.delete(uuid)
|
126
|
+
hostname, port = server['addr'].split(':')
|
127
|
+
# Service has stopped and needs to be removed
|
128
|
+
remove_server(File.join(server['name'], server['version'].to_s, server['region']), hostname, port, true)
|
117
129
|
end
|
118
130
|
end
|
119
131
|
|
120
132
|
# Service information created
|
121
133
|
def service_info_created(path, value=nil)
|
122
|
-
logger.
|
134
|
+
logger.trace("service_info_created: #{path}", value)
|
123
135
|
# path: "uuid/key"
|
124
136
|
uuid, key = path.split('/')
|
125
137
|
|
@@ -135,7 +147,7 @@ module RubySkynet
|
|
135
147
|
|
136
148
|
# Service information changed
|
137
149
|
def service_info_updated(path, value=nil)
|
138
|
-
logger.
|
150
|
+
logger.trace("service_info_updated: #{path}", value)
|
139
151
|
# path: "uuid/key"
|
140
152
|
uuid, key = path.split('/')
|
141
153
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby_skynet
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-08-
|
11
|
+
date: 2013-08-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: semantic_logger
|