ruby_skynet 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f15ead62305cae332e890c1ad63a3f11e9085ffe
4
- data.tar.gz: aa3243ae8229d4cd7a5f82ac673fd3e7cc700b17
3
+ metadata.gz: 5e4070e64ce70f32032c82eee9e6af0eaa94626c
4
+ data.tar.gz: 62ff2d291f403ac51406e17ead44a3f5feacd120
5
5
  SHA512:
6
- metadata.gz: 363ed46b3ee9182fa572fb00724031f01d077600a304ac3d38ccbbb023b3c33ff25872c72b9eeb7c130e4784ce9c6c2ff807a9c522d391ef31a8de9564fdd7e3
7
- data.tar.gz: 4eeb6d97725af7fb087e40afbc8b8d459c8889b84c1d4b1756885eedc3322adbf22a5afd89d44c97ad456e7441e0dd09fb90007c1a28204835638ab1e9404faf
6
+ metadata.gz: 66559b704a0e6f1f73967e555d23c6f3eb975c58059324cb9f06944b4eff65ae22105ce1a28f4e7533d9cdb9a84d3f20de391dce860c2aa0a58bc935a28fbf88
7
+ data.tar.gz: 17d54811347c21ff8c20a174923ce96469e05ed56c9603abef0d1a3374eda8a80e1536cb3f35cbc07746ccfae7b9dac9afb126eb7630e900024fea863ae26687
data/Gemfile CHANGED
@@ -2,18 +2,19 @@ source :rubygems
2
2
 
3
3
  group :test do
4
4
  gem "shoulda"
5
- gem "mocha", :require => false
6
5
  end
7
6
 
8
7
  gem "rake"
9
8
  gem "semantic_logger"
10
9
  gem "resilient_socket"
10
+ # Doozer Client
11
+ gem "ruby_doozer"
11
12
  # Thread Safe Hash and Array
12
13
  gem "thread_safe"
14
+ # Connection pool
13
15
  gem "gene_pool"
14
16
  # For looking up Service entries in Doozer
15
17
  gem "multi_json"
16
- gem "ruby_protobuf"
17
18
  # Wire format when communicating with services
18
19
  gem "bson"
19
20
  gem "bson_ext", :platform => :ruby
data/Gemfile.lock CHANGED
@@ -4,8 +4,8 @@ GEM
4
4
  activesupport (3.2.13)
5
5
  i18n (= 0.6.1)
6
6
  multi_json (~> 1.0)
7
- atomic (1.0.1)
8
- atomic (1.0.1-java)
7
+ atomic (1.0.2)
8
+ atomic (1.0.2-java)
9
9
  bourne (1.4.0)
10
10
  mocha (~> 0.13.2)
11
11
  bson (1.8.3)
@@ -21,6 +21,11 @@ GEM
21
21
  rake (10.0.3)
22
22
  resilient_socket (0.4.0)
23
23
  semantic_logger
24
+ ruby_doozer (0.4.0)
25
+ gene_pool
26
+ resilient_socket
27
+ ruby_protobuf
28
+ semantic_logger
24
29
  ruby_protobuf (0.4.11)
25
30
  semantic_logger (2.0.0)
26
31
  sync_attr
@@ -44,11 +49,10 @@ DEPENDENCIES
44
49
  bson
45
50
  bson_ext
46
51
  gene_pool
47
- mocha
48
52
  multi_json
49
53
  rake
50
54
  resilient_socket
51
- ruby_protobuf
55
+ ruby_doozer
52
56
  semantic_logger
53
57
  shoulda
54
58
  thread_safe
data/Rakefile CHANGED
@@ -24,12 +24,12 @@ task :gem do |t|
24
24
  spec.files = FileList["./**/*"].exclude(/\.gem$/, /\.log$/,/nbproject/).map{|f| f.sub(/^\.\//, '')}
25
25
  spec.license = "Apache License V2.0"
26
26
  spec.has_rdoc = true
27
- spec.add_dependency 'semantic_logger'
28
- spec.add_dependency 'resilient_socket'
27
+ spec.add_dependency 'semantic_logger', '>= 2.0.0'
28
+ spec.add_dependency 'resilient_socket', '>= 0.4.0'
29
29
  spec.add_dependency 'multi_json', '>= 1.6.1'
30
- spec.add_dependency 'bson'
31
- spec.add_dependency 'ruby_protobuf'
32
- spec.add_dependency 'gene_pool'
30
+ spec.add_dependency 'bson', '>= 1.5.2'
31
+ spec.add_dependency 'ruby_doozer', '>= 0.5.0'
32
+ spec.add_dependency 'gene_pool', '>= 1.3.0'
33
33
  end
34
34
  Gem::Package.build gemspec
35
35
  end
@@ -7,10 +7,6 @@ require 'ruby_skynet'
7
7
  SemanticLogger::Logger.default_level = :trace
8
8
  SemanticLogger::Logger.appenders << SemanticLogger::Appender::File.new('echo_server.log')
9
9
 
10
- # Specify Port and Hostname to listen for requests on
11
- RubySkynet::Server.port = 2020
12
- RubySkynet::Server.hostname = '127.0.0.1'
13
-
14
10
  # Just echo back any parameters received when the echo method is called
15
11
  class EchoService
16
12
  include RubySkynet::Service
@@ -1,10 +1,10 @@
1
1
  # Ruby Skynet Client & Server Configuration Parameters
2
2
  # :region [String]
3
- # Region name to use for service lookup
3
+ # Region name to use for service lookups
4
4
  # Default: Rails.env
5
5
  #
6
6
  # :services_path [String]
7
- # Path within to look for service implementations that will be loaded
7
+ # Path within which to look for service implementations that will be loaded
8
8
  # when RubySkynet::Server.load_services is called
9
9
  #
10
10
  # :server_port [Integer]
@@ -53,6 +53,10 @@
53
53
  # Randomly select a server from the list every time a connection
54
54
  # is established, including during automatic connection recovery.
55
55
  # Default: :ordered
56
+ #
57
+ # :pool_size [Integer]
58
+ # Maximum size of the connection pool to doozer
59
+ # Default: 10
56
60
  defaults: &defaults
57
61
  :services_path: app/services
58
62
  :server_port: 2000
@@ -64,6 +68,11 @@ defaults: &defaults
64
68
  :connect_retry_count: 10
65
69
  :connect_retry_interval: 0.5
66
70
  :server_selector: :random
71
+ # Doozer Connection Pool settings
72
+ :pool_size: 10
73
+ :pool_timeout: 30
74
+ :pool_warn_timeout: 2
75
+ :pool_idle_timeout: 600
67
76
 
68
77
  development:
69
78
  <<: *defaults
@@ -61,7 +61,7 @@ module RubySkynet
61
61
  retries = 0
62
62
  # If it cannot connect to a server, try a different server
63
63
  begin
64
- Connection.with_connection(Registry.server_for(@skynet_name, @version, @region), connection_params) do |connection|
64
+ Connection.with_connection(::RubySkynet.services.server_for(@skynet_name, @version, @region), connection_params) do |connection|
65
65
  connection.rpc_call(request_id, @skynet_name, method_name, parameters)
66
66
  end
67
67
  rescue ResilientSocket::ConnectionFailure => exc
@@ -1,3 +1,4 @@
1
+ require 'bson'
1
2
  require 'socket'
2
3
 
3
4
  module RubySkynet
@@ -5,7 +5,7 @@ require 'resilient_socket'
5
5
  require 'sync_attr'
6
6
 
7
7
  #
8
- # RubySkynet Connection
8
+ # RubySkynet Client Connection
9
9
  #
10
10
  # Handles connecting to Skynet Servers as a host:port pair
11
11
  #
@@ -237,7 +237,7 @@ module RubySkynet
237
237
  end
238
238
 
239
239
  # Cleanup corresponding connection pool when a server terminates
240
- Registry.on_server_removed(server) do
240
+ RubySkynet.services.on_server_removed(server) do
241
241
  pool = @@connection_pools.delete(server)
242
242
  # Cannot close all the connections since they could still be in use
243
243
  pool.remove_idle(0) if pool
@@ -16,7 +16,7 @@ namespace :ruby_skynet do
16
16
  end
17
17
 
18
18
  # Connect to doozer
19
- RubySkynet::Registry.service_registry
19
+ RubySkynet.services
20
20
 
21
21
  RubySkynet::Server.load_services
22
22
 
@@ -1,4 +1,7 @@
1
+ require 'sync_attr'
2
+
1
3
  module RubySkynet
4
+ include SyncAttr
2
5
 
3
6
  # Returns the default region for all Ruby Skynet Clients and Services
4
7
  def self.region
@@ -42,6 +45,50 @@ module RubySkynet
42
45
  @@local_ip_address = local_ip_address
43
46
  end
44
47
 
48
+ # Returns the services registry consisting of service names
49
+ # and the hosts on which they are running
50
+ sync_cattr_reader :services do
51
+ ServiceRegistry.new(
52
+ :root_path => "/services",
53
+ :doozer => doozer_config
54
+ )
55
+ end
56
+
57
+ # Default doozer configuration
58
+ # To replace this default, set the config as follows:
59
+ # RubySkynet::Client.doozer_config = { .... }
60
+ #
61
+ # :servers [Array of String]
62
+ # Array of URL's of doozer servers to connect to with port numbers
63
+ # ['server1:2000', 'server2:2000']
64
+ #
65
+ # The second server will only be attempted once the first server
66
+ # cannot be connected to or has timed out on connect
67
+ # A read failure or timeout will not result in switching to the second
68
+ # server, only a connection failure or during an automatic reconnect
69
+ #
70
+ # :read_timeout [Float]
71
+ # Time in seconds to timeout on read
72
+ # Can be overridden by supplying a timeout in the read call
73
+ #
74
+ # :connect_timeout [Float]
75
+ # Time in seconds to timeout when trying to connect to the server
76
+ #
77
+ # :connect_retry_count [Fixnum]
78
+ # Number of times to retry connecting when a connection fails
79
+ #
80
+ # :connect_retry_interval [Float]
81
+ # Number of seconds between connection retry attempts after the first failed attempt
82
+ sync_cattr_accessor :doozer_config do
83
+ {
84
+ :servers => ['127.0.0.1:8046'],
85
+ :read_timeout => 5,
86
+ :connect_timeout => 3,
87
+ :connect_retry_interval => 1,
88
+ :connect_retry_count => 30
89
+ }
90
+ end
91
+
45
92
  # Load the Encryption Configuration from a YAML file
46
93
  # filename:
47
94
  # Name of file to read.
@@ -61,7 +108,7 @@ module RubySkynet
61
108
  RubySkynet.services_path = cfg.delete(:services_path) if [:services_path]
62
109
  RubySkynet.server_port = cfg.delete(:server_port) if [:server_port]
63
110
  RubySkynet.local_ip_address = cfg.delete(:local_ip_address) if [:local_ip_address]
64
- RubySkynet::Registry.doozer_config = cfg.delete(:doozer) if [:doozer]
111
+ RubySkynet.doozer_config = cfg.delete(:doozer) if [:doozer]
65
112
 
66
113
  cfg.each_pair {|k,v| RubySkynet::Server.logger.warn "Ignoring unknown RubySkynet config option #{k} => #{v}"}
67
114
  end
@@ -1,4 +1,4 @@
1
- require 'bson'
1
+ require 'thread_safe'
2
2
 
3
3
  #
4
4
  # RubySkynet Server
@@ -206,13 +206,13 @@ module RubySkynet
206
206
  # Registers a Service Class as being available at this server
207
207
  def register_service(klass)
208
208
  logger.debug "Registering Service: #{klass.name} with name: #{klass.skynet_name}"
209
- Registry.register_service(klass.skynet_name, klass.skynet_version || 1, klass.skynet_region, @hostname, @port)
209
+ ::RubySkynet.services.register_service(klass.skynet_name, klass.skynet_version || 1, klass.skynet_region, @hostname, @port)
210
210
  end
211
211
 
212
212
  # De-register service from this server
213
213
  def deregister_service(klass)
214
214
  logger.debug "De-registering Service: #{klass.name} with name: #{klass.skynet_name}"
215
- Registry.deregister_service(klass.skynet_name, klass.skynet_version || 1, klass.skynet_region, @hostname, @port)
215
+ ::RubySkynet.services.deregister_service(klass.skynet_name, klass.skynet_version || 1, klass.skynet_region, @hostname, @port)
216
216
  end
217
217
 
218
218
  # Called for each message received from the client
@@ -0,0 +1,248 @@
1
+ require 'sync_attr'
2
+ require 'multi_json'
3
+ require 'thread_safe'
4
+ require 'gene_pool'
5
+ require 'resolv'
6
+ require 'ruby_doozer'
7
+
8
+ #
9
+ # RubySkynet Sevices Registry
10
+ #
11
+ # Based on the Skynet Services Registry, obtains and keeps up to date a list of
12
+ # all services and which servers they are available on.
13
+ #
14
+ module RubySkynet
15
+ class ServiceRegistry < RubyDoozer::Registry
16
+ include SyncAttr
17
+ include SemanticLogger::Loggable
18
+
19
+ IPV4_REG_EXP = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/
20
+
21
+ # Returns [Integer] the score for the supplied ip_address
22
+ # Score currently ranges from 0 to 4 with 4 being the best score
23
+ # If the IP address does not match an IP v4 address a DNS lookup will
24
+ # be performed
25
+ def self.score_for_server(ip_address, local_ip_address)
26
+ score = 0
27
+ # Each matching element adds 1 to the score
28
+ # 192.168. 0. 0
29
+ # 1
30
+ # 1
31
+ # 1
32
+ # 1
33
+ server_match = IPV4_REG_EXP.match(ip_address) || IPV4_REG_EXP.match(Resolv::DNS.new.getaddress(ip_address).to_s)
34
+ if server_match
35
+ local_match = IPV4_REG_EXP.match(local_ip_address)
36
+ score = 0
37
+ (1..4).each do |i|
38
+ break if local_match[i].to_i != server_match[i].to_i
39
+ score += 1
40
+ end
41
+ end
42
+ score
43
+ end
44
+
45
+ # Create a service registry
46
+ # See: RubyDoozer::Registry for the parameters
47
+ def initialize(params)
48
+ super
49
+
50
+ # Registry has the following format
51
+ # Key: [String] 'name/version/region'
52
+ # Value: [Array<String>] 'host:port', 'host:port'
53
+ @registry = ThreadSafe::Hash.new
54
+
55
+ path = "#{@root_path}/**"
56
+ doozer_pool.with_connection do |doozer|
57
+ @current_revision = doozer.current_revision
58
+ # Fetch all the configuration information from Doozer and set the internal copy
59
+ doozer.walk(path, @current_revision) do |path, value|
60
+ service_info_changed(relative_path(path), value)
61
+ end
62
+ end
63
+
64
+ # Start monitoring thread
65
+ monitor_thread
66
+
67
+ # Register Callbacks
68
+ on_update {|path, value| service_info_changed(path, value) }
69
+ on_delete {|path, value| service_info_changed(path) }
70
+ end
71
+
72
+ # Returns the Service Registry as a Hash
73
+ def to_h
74
+ @registry.dup
75
+ end
76
+
77
+ # Register the supplied service at this Skynet Server host and Port
78
+ def register_service(name, version, region, hostname, port)
79
+ config = {
80
+ "Config" => {
81
+ "UUID" => "#{hostname}:#{port}-#{$$}-#{name}-#{version}",
82
+ "Name" => name,
83
+ "Version" => version.to_s,
84
+ "Region" => region,
85
+ "ServiceAddr" => {
86
+ "IPAddress" => hostname,
87
+ "Port" => port,
88
+ "MaxPort" => port + 999
89
+ },
90
+ },
91
+ "Registered" => true
92
+ }
93
+ self["#{name}/#{version}/#{region}/#{hostname}/#{port}"] = MultiJson.encode(config)
94
+ end
95
+
96
+ # Deregister the supplied service from the Registry
97
+ def deregister_service(name, version, region, hostname, port)
98
+ delete("#{name}/#{version}/#{region}/#{hostname}/#{port}")
99
+ end
100
+
101
+ # Return a server that implements the specified service
102
+ def server_for(name, version='*', region='Development')
103
+ if servers = servers_for(name, version, region)
104
+ # Randomly select one of the servers offering the service
105
+ servers[rand(servers.size)]
106
+ else
107
+ msg = "No servers available for service: #{name} with version: #{version} in region: #{region}"
108
+ logger.warn msg
109
+ raise ServiceUnavailable.new(msg)
110
+ end
111
+ end
112
+
113
+ # Returns [Array<String>] a list of servers implementing the requested service
114
+ def servers_for(name, version='*', region='Development')
115
+ if version == '*'
116
+ # Find the highest version for the named service in this region
117
+ version = -1
118
+ @registry.keys.each do |key|
119
+ if match = key.match(/#{name}\/(\d+)\/#{region}/)
120
+ ver = match[1].to_i
121
+ version = ver if ver > version
122
+ end
123
+ end
124
+ end
125
+ if server_infos = @registry["#{name}/#{version}/#{region}"]
126
+ server_infos.first.servers
127
+ end
128
+ end
129
+
130
+ # Invokes registered callbacks when a specific server is shutdown or terminates
131
+ # Not when a server de-registers itself
132
+ # The callback will only be called once and will need to be re-registered
133
+ # after being called if future callbacks are required for that server
134
+ def on_server_removed(server, &block)
135
+ ((@on_server_removed_callbacks ||= ThreadSafe::Hash.new)[server] ||= ThreadSafe::Array.new) << block
136
+ end
137
+
138
+ ############################
139
+ protected
140
+
141
+ # Service information changed in doozer, so update internal registry
142
+ def service_info_changed(path, value=nil)
143
+ # path: "TutorialService/1/Development/127.0.0.1/9000"
144
+ e = path.split('/')
145
+
146
+ # Key: [String] 'name/version/region'
147
+ key = "#{e[0]}/#{e[1]}/#{e[2]}"
148
+ hostname, port = e[3], e[4]
149
+
150
+ if value
151
+ entry = MultiJson.load(value)
152
+ if entry['Registered']
153
+ add_server(key, hostname, port)
154
+ else
155
+ # Service just de-registered
156
+ remove_server(key, hostname, port, false)
157
+ end
158
+ else
159
+ # Service has stopped and needs to be removed
160
+ remove_server(key, hostname, port, true)
161
+ end
162
+ end
163
+
164
+ # :score: [Integer] Score
165
+ # :servers: [Array<String>] 'host:port', 'host:port'
166
+ ServerInfo = Struct.new(:score, :servers )
167
+
168
+ # Format of the internal services registry
169
+ # key: [String] "<name>/<version>/<region>"
170
+ # value: [ServiceInfo, ServiceInfo]
171
+ # Sorted by highest score first
172
+
173
+ # Add the host to the registry based on it's score
174
+ def add_server(key, hostname, port)
175
+ server = "#{hostname}:#{port}"
176
+ logger.debug "#monitor Add/Update Service: #{key} => #{server.inspect}"
177
+
178
+ server_infos = (@registry[key] ||= ThreadSafe::Array.new)
179
+
180
+ # If already present, then nothing to do
181
+ server_info = server_infos.find{|si| si.servers.include?(server)}
182
+ return server_info if server_info
183
+
184
+ # Look for the same score with a different server
185
+ score = self.class.score_for_server(hostname, RubySkynet.local_ip_address)
186
+ if server_info = server_infos.find{|si| si.score == score}
187
+ server_info.servers << server
188
+ return server_info
189
+ end
190
+
191
+ # New score
192
+ servers = ThreadSafe::Array.new
193
+ servers << server
194
+ server_info = ServerInfo.new(score, servers)
195
+
196
+ # Insert into Array in order of score
197
+ if index = server_infos.find_index {|si| si.score <= score}
198
+ server_infos.insert(index, server_info)
199
+ else
200
+ server_infos << server_info
201
+ end
202
+ server_info
203
+ end
204
+
205
+ # Remove the host from the registry based
206
+ # Returns the server instance if it was removed
207
+ def remove_server(key, hostname, port, notify)
208
+ server = "#{hostname}:#{port}"
209
+ logger.debug "Remove Service: #{key} => #{server.inspect}"
210
+ server_info = nil
211
+ if server_infos = @registry[key]
212
+ server_infos.each do |si|
213
+ if si.servers.delete(server)
214
+ server_info = si
215
+ break
216
+ end
217
+ end
218
+
219
+ # Found server
220
+ if server_info
221
+ # Cleanup if no more servers in server list
222
+ server_infos.delete(server_info) if server_info.servers.size == 0
223
+
224
+ # Cleanup if no more server infos
225
+ @registry.delete(key) if server_infos.size == 0
226
+
227
+ server_removed(server) if notify
228
+ end
229
+ end
230
+ server_info
231
+ end
232
+
233
+ # Invoke any registered callbacks for the specific server
234
+ def server_removed(server)
235
+ if @on_server_removed_callbacks && (callbacks = @on_server_removed_callbacks.delete(server))
236
+ callbacks.each do |block|
237
+ begin
238
+ logger.info "Calling callback for server: #{server}"
239
+ block.call(server)
240
+ rescue Exception => exc
241
+ logger.error("Exception during a callback for server: #{server}", exc)
242
+ end
243
+ end
244
+ end
245
+ end
246
+
247
+ end
248
+ end
@@ -1,3 +1,3 @@
1
1
  module RubySkynet #:nodoc
2
- VERSION = "0.5.0"
2
+ VERSION = "0.6.0"
3
3
  end
data/lib/ruby_skynet.rb CHANGED
@@ -2,17 +2,16 @@ require 'semantic_logger'
2
2
  require 'ruby_skynet/exceptions'
3
3
  require 'ruby_skynet/version'
4
4
  require 'ruby_skynet/ruby_skynet'
5
+ require 'ruby_doozer'
6
+
5
7
  module RubySkynet
6
- module Doozer
7
- autoload :Client, 'ruby_skynet/doozer/client'
8
- end
9
- autoload :Registry, 'ruby_skynet/registry'
10
- autoload :Connection, 'ruby_skynet/connection'
11
- autoload :Base, 'ruby_skynet/base'
12
- autoload :Common, 'ruby_skynet/common'
13
- autoload :Client, 'ruby_skynet/client'
14
- autoload :Service, 'ruby_skynet/service'
15
- autoload :Server, 'ruby_skynet/server'
8
+ autoload :Base, 'ruby_skynet/base'
9
+ autoload :Common, 'ruby_skynet/common'
10
+ autoload :Connection, 'ruby_skynet/connection'
11
+ autoload :Client, 'ruby_skynet/client'
12
+ autoload :Service, 'ruby_skynet/service'
13
+ autoload :Server, 'ruby_skynet/server'
14
+ autoload :ServiceRegistry, 'ruby_skynet/service_registry'
16
15
  end
17
16
 
18
17
  if defined?(Rails)
data/test/client_test.rb CHANGED
@@ -58,6 +58,8 @@ class ClientTest < Test::Unit::TestCase
58
58
  @region = 'ClientTest'
59
59
  RubySkynet.region = @region
60
60
  RubySkynet::Server.start
61
+ # Give doozer time to push out the presence of the service above
62
+ sleep 0.1
61
63
 
62
64
  @service_name = 'ClientTestService'
63
65
  @version = 1