scales-worker 0.0.1.beta.2 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/scales-worker/boot/autoload.rb +1 -0
- data/lib/scales-worker/pusher.rb +15 -3
- data/lib/scales-worker/status.rb +12 -9
- data/lib/scales-worker/version.rb +1 -1
- data/lib/scales-worker/worker.rb +37 -18
- metadata +20 -20
data/lib/scales-worker/pusher.rb
CHANGED
@@ -24,9 +24,9 @@ module Scales
|
|
24
24
|
env = Path.with_options_to_env(path)
|
25
25
|
|
26
26
|
response = @app.call(env)
|
27
|
-
response.last.close
|
27
|
+
response.last.close if response.last.respond_to?(:close)
|
28
28
|
|
29
|
-
|
29
|
+
push_response(path, Response.to_string(response)) if path[:push]
|
30
30
|
|
31
31
|
env
|
32
32
|
end
|
@@ -37,7 +37,7 @@ module Scales
|
|
37
37
|
|
38
38
|
begin
|
39
39
|
response = @app.call(request)
|
40
|
-
response.last.close
|
40
|
+
response.last.close if response.last.respond_to?(:close)
|
41
41
|
rescue Exception => e
|
42
42
|
puts e
|
43
43
|
end
|
@@ -70,6 +70,18 @@ module Scales
|
|
70
70
|
puts "Pushing paths: #{progress}% (#{@done}/#{@total})".green
|
71
71
|
puts "Done.".green
|
72
72
|
end
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def push_response(path, response)
|
77
|
+
Storage::Sync.set_content(path[:to], response)
|
78
|
+
data = {
|
79
|
+
:path => path[:to],
|
80
|
+
:format => path[:format].to_s.upcase,
|
81
|
+
:type => "push_#{Cache.resource_or_partial?(path[:to])}"
|
82
|
+
}
|
83
|
+
Storage::Sync.connection.publish "scales_monitor_events", data.to_json
|
84
|
+
end
|
73
85
|
end
|
74
86
|
end
|
75
87
|
end
|
data/lib/scales-worker/status.rb
CHANGED
@@ -2,12 +2,15 @@ module Scales
|
|
2
2
|
module Worker
|
3
3
|
|
4
4
|
class Status
|
5
|
-
attr_reader :key, :id, :address, :port
|
5
|
+
attr_reader :key, :id, :address, :port, :logger, :log_path
|
6
6
|
|
7
7
|
def initialize address, port = nil
|
8
|
-
@id
|
9
|
-
@key
|
8
|
+
@id = SecureRandom.hex(8)
|
9
|
+
@key = "scales_worker_#{@id}"
|
10
10
|
@address, @port = address.to_s, port.to_s
|
11
|
+
@log_path = "log/scales_worker.#{@id}.log"
|
12
|
+
@logger = Logger.new(Scales.env == "test" ? STDOUT : @log_path)
|
13
|
+
@redis = Scales::Storage::Sync.new_connection!
|
11
14
|
end
|
12
15
|
|
13
16
|
def start!
|
@@ -22,8 +25,8 @@ module Scales
|
|
22
25
|
}
|
23
26
|
json = JSON.generate(data)
|
24
27
|
|
25
|
-
|
26
|
-
|
28
|
+
@redis.set(@key, json)
|
29
|
+
@redis.publish("scales_monitor_events", json)
|
27
30
|
@already_stopped = false
|
28
31
|
end
|
29
32
|
|
@@ -36,8 +39,8 @@ module Scales
|
|
36
39
|
:type => "worker_stopped"
|
37
40
|
}
|
38
41
|
json = JSON.generate(data)
|
39
|
-
|
40
|
-
|
42
|
+
@redis.del(@key)
|
43
|
+
@redis.publish("scales_monitor_events", json)
|
41
44
|
@already_stopped = true
|
42
45
|
end
|
43
46
|
|
@@ -50,7 +53,7 @@ module Scales
|
|
50
53
|
:method => job['REQUEST_METHOD']
|
51
54
|
}
|
52
55
|
json = JSON.generate(data)
|
53
|
-
|
56
|
+
@redis.publish("scales_monitor_events", json)
|
54
57
|
end
|
55
58
|
|
56
59
|
def put_response_in_queue!(response)
|
@@ -63,7 +66,7 @@ module Scales
|
|
63
66
|
:status => response[0]
|
64
67
|
}
|
65
68
|
json = JSON.generate(data)
|
66
|
-
|
69
|
+
@redis.publish("scales_monitor_events", json)
|
67
70
|
end
|
68
71
|
|
69
72
|
end
|
data/lib/scales-worker/worker.rb
CHANGED
@@ -3,9 +3,11 @@ module Scales
|
|
3
3
|
class Worker
|
4
4
|
attr_reader :app
|
5
5
|
attr_reader :type
|
6
|
+
attr_reader :pool
|
7
|
+
attr_reader :status
|
6
8
|
|
7
9
|
def initialize(type = Application::Rails)
|
8
|
-
@type, @app, @status = type, type.app, Status.new("localhost")
|
10
|
+
@type, @app, @status, @pool = type, type.app, Status.new("localhost"), []
|
9
11
|
at_exit{ @status.stop! }
|
10
12
|
end
|
11
13
|
|
@@ -21,10 +23,10 @@ module Scales
|
|
21
23
|
|
22
24
|
begin
|
23
25
|
response = @app.call(env)
|
24
|
-
response.last.close
|
26
|
+
response.last.close if response.last.respond_to?(:close)
|
25
27
|
[id, Response.to_job(id, response)]
|
26
|
-
rescue
|
27
|
-
[id, [500, {},
|
28
|
+
rescue Exception => e
|
29
|
+
[id, [500, {}, e.to_s]]
|
28
30
|
end
|
29
31
|
end
|
30
32
|
|
@@ -35,7 +37,7 @@ module Scales
|
|
35
37
|
|
36
38
|
begin
|
37
39
|
response = @app.call(request)
|
38
|
-
response.last.close
|
40
|
+
response.last.close if response.last.respond_to?(:close)
|
39
41
|
rescue Exception => e
|
40
42
|
puts e
|
41
43
|
end
|
@@ -43,24 +45,38 @@ module Scales
|
|
43
45
|
end
|
44
46
|
|
45
47
|
# Wait for a request, process it, publish the response and exit
|
46
|
-
def process_request!
|
47
|
-
job = Scales::
|
48
|
+
def process_request!
|
49
|
+
job = Thread.current[:redis_blocking].brpop(Scales::Storage::REQUEST_QUEUE, 0).last
|
48
50
|
id, response = nil, nil
|
49
51
|
|
50
|
-
Thread.
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
@status.put_response_in_queue!(response)
|
56
|
-
Scales::PubSub::Sync.publish("scales_response_#{id}", JSON.generate(response))
|
57
|
-
end
|
58
|
-
|
59
|
-
thread.join if should_wait_for_request_to_finish
|
52
|
+
Thread.current[:post_process_queue] = []
|
53
|
+
id, response = process!(job)
|
54
|
+
post_process!(job)
|
55
|
+
@status.put_response_in_queue!(response)
|
56
|
+
Thread.current[:redis_nonblocking].publish(Scales::Storage::RESPONSE_CHANNEL, JSON.generate(response))
|
60
57
|
|
61
58
|
[id, response]
|
62
59
|
end
|
63
60
|
|
61
|
+
def start_pool!(size = Scales.config.worker_threads)
|
62
|
+
Thread.abort_on_exception = true
|
63
|
+
size.times do
|
64
|
+
@pool << Thread.new do
|
65
|
+
Thread.current[:redis_blocking] = Scales::Storage::Sync.new_connection!
|
66
|
+
Thread.current[:redis_nonblocking] = Scales::Storage::Sync.new_connection!
|
67
|
+
loop do
|
68
|
+
begin
|
69
|
+
process_request!
|
70
|
+
rescue Exception => e
|
71
|
+
@status.logger.error(e.to_s)
|
72
|
+
raise e if Scales.env == "test"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
sleep
|
78
|
+
end
|
79
|
+
|
64
80
|
# Loop the processing of requests
|
65
81
|
def work!
|
66
82
|
@status.start!
|
@@ -68,11 +84,14 @@ module Scales
|
|
68
84
|
puts "Environment: #{Scales.env}".green
|
69
85
|
puts "Application: #{@type.name}".green
|
70
86
|
puts "Path: #{Dir.pwd}".green
|
87
|
+
puts "Log Path: #{@status.log_path}".green
|
88
|
+
puts "Threads: #{Scales.config.worker_threads}".green
|
71
89
|
puts "Redis: #{Scales.config.host}:#{Scales.config.port}/#{Scales.config.database}".green
|
72
90
|
|
73
91
|
begin
|
74
|
-
|
92
|
+
start_pool!
|
75
93
|
rescue Interrupt => e
|
94
|
+
@pool.map(&:exit)
|
76
95
|
puts "Goodbye".green
|
77
96
|
end
|
78
97
|
end
|
metadata
CHANGED
@@ -1,30 +1,30 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scales-worker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.4
|
5
|
+
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Thomas Fankhauser
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-08-
|
12
|
+
date: 2012-08-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: scales-core
|
16
|
-
requirement: &
|
16
|
+
requirement: &70342046065420 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - =
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: 0.0.
|
21
|
+
version: 0.0.4
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70342046065420
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: rails
|
27
|
-
requirement: &
|
27
|
+
requirement: &70342046064960 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 3.2.6
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70342046064960
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: sqlite3
|
38
|
-
requirement: &
|
38
|
+
requirement: &70342046064460 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: 1.3.6
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70342046064460
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: rspec
|
49
|
-
requirement: &
|
49
|
+
requirement: &70342046064000 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: '2.11'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70342046064000
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: rake
|
60
|
-
requirement: &
|
60
|
+
requirement: &70342046063540 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ! '>='
|
@@ -65,10 +65,10 @@ dependencies:
|
|
65
65
|
version: 0.9.2.2
|
66
66
|
type: :runtime
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *70342046063540
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: nokogiri
|
71
|
-
requirement: &
|
71
|
+
requirement: &70342046063080 !ruby/object:Gem::Requirement
|
72
72
|
none: false
|
73
73
|
requirements:
|
74
74
|
- - ! '>='
|
@@ -76,10 +76,10 @@ dependencies:
|
|
76
76
|
version: 1.5.5
|
77
77
|
type: :runtime
|
78
78
|
prerelease: false
|
79
|
-
version_requirements: *
|
79
|
+
version_requirements: *70342046063080
|
80
80
|
- !ruby/object:Gem::Dependency
|
81
81
|
name: jsonpath
|
82
|
-
requirement: &
|
82
|
+
requirement: &70342046062620 !ruby/object:Gem::Requirement
|
83
83
|
none: false
|
84
84
|
requirements:
|
85
85
|
- - ! '>='
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
version: 0.5.0
|
88
88
|
type: :runtime
|
89
89
|
prerelease: false
|
90
|
-
version_requirements: *
|
90
|
+
version_requirements: *70342046062620
|
91
91
|
description: Super Scale Caching Framework - Worker
|
92
92
|
email:
|
93
93
|
- tommylefunk@googlemail.com
|
@@ -138,9 +138,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
138
138
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
139
139
|
none: false
|
140
140
|
requirements:
|
141
|
-
- - ! '
|
141
|
+
- - ! '>='
|
142
142
|
- !ruby/object:Gem::Version
|
143
|
-
version:
|
143
|
+
version: '0'
|
144
144
|
requirements: []
|
145
145
|
rubyforge_project:
|
146
146
|
rubygems_version: 1.8.11
|