son_of_a_batch 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.rspec +1 -0
- data/Gemfile +22 -0
- data/Gemfile.lock +66 -0
- data/LICENSE.txt +20 -0
- data/README.textile +65 -0
- data/Rakefile +43 -0
- data/VERSION +1 -0
- data/app/endpoints/sleepy.rb +80 -0
- data/app/views/debug.haml +4 -0
- data/app/views/layout.haml +39 -0
- data/app/views/root.haml +28 -0
- data/config/bootstrap.rb +114 -0
- data/config/son_of_a_batch-private.example.rb +8 -0
- data/config/son_of_a_batch-private.rb +5 -0
- data/config/son_of_a_batch.rb +19 -0
- data/lib/boot.rb +72 -0
- data/lib/son_of_a_batch.rb +0 -0
- data/public/stylesheets/style.css +296 -0
- data/son_of_a_batch.gemspec +96 -0
- data/son_of_a_batch.rb +153 -0
- data/spec/son_of_a_batch_spec.rb +7 -0
- data/spec/spec_helper.rb +25 -0
- metadata +203 -0
data/son_of_a_batch.rb
ADDED
@@ -0,0 +1,153 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$:<< './lib'
|
3
|
+
|
4
|
+
# A simple dashboard for
|
5
|
+
#
|
6
|
+
# See
|
7
|
+
# app/views -- templates
|
8
|
+
# public -- static files
|
9
|
+
# config/son_of_a_batch.rb -- configuration
|
10
|
+
#
|
11
|
+
|
12
|
+
require 'boot'
|
13
|
+
require 'gorillib'
|
14
|
+
require 'tilt'
|
15
|
+
require 'yajl/json_gem'
|
16
|
+
|
17
|
+
require 'goliath'
|
18
|
+
require 'goliath/rack/templates'
|
19
|
+
require 'goliath/plugins/latency'
|
20
|
+
require 'em-synchrony/em-http'
|
21
|
+
require 'rack/abstract_format'
|
22
|
+
|
23
|
+
class SonOfABatch < Goliath::API
|
24
|
+
use Goliath::Rack::Params # parse query & body params
|
25
|
+
use Goliath::Rack::Formatters::JSON # JSON output formatter
|
26
|
+
use Goliath::Rack::Render # auto-negotiate response format
|
27
|
+
use Rack::AbstractFormat, 'application/json'
|
28
|
+
|
29
|
+
include Goliath::Rack::Templates # render templated files from ./views
|
30
|
+
use(Rack::Static, # render static files from ./public
|
31
|
+
:root => Goliath::Application.root_path("public"), :urls => ["/favicon.ico", '/stylesheets', '/javascripts', '/images'])
|
32
|
+
# plugin Goliath::Plugin::Latency # ask eventmachine reactor to track its latency
|
33
|
+
|
34
|
+
TARGET_CONCURRENCY = 10
|
35
|
+
QUERIES = [ 1.0, 60.5, 2.5, 0.5, 1.0, 0.25, 1.0, 60.5, 2.5, 0.5, 1.0, 0.25 ]
|
36
|
+
|
37
|
+
def recent_latency
|
38
|
+
Goliath::Plugin::Latency.recent_latency if defined?(Goliath::Plugin::Latency)
|
39
|
+
end
|
40
|
+
|
41
|
+
def response(env)
|
42
|
+
batch_id = "%7.04f" % (env[:start_time].to_f - 100*(env[:start_time].to_f.to_i / 100))
|
43
|
+
case env['PATH_INFO']
|
44
|
+
when '/' then return [200, {}, haml(:root)]
|
45
|
+
when '/debug' then return [200, {}, haml(:debug)]
|
46
|
+
when '/get' then :pass
|
47
|
+
else raise Goliath::Validation::NotFoundError
|
48
|
+
end
|
49
|
+
|
50
|
+
env.logger.debug "req #{object_id} @#{batch_id}: constructing request group"
|
51
|
+
BatchIterator.new(env, batch_id, QUERIES.each_with_index.to_a, TARGET_CONCURRENCY).perform
|
52
|
+
env.logger.debug "req #{object_id} @#{batch_id}: constructed request group"
|
53
|
+
chunked_streaming_response(200, {'X-Responder' => self.class.to_s })
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
class BatchIterator < EM::Synchrony::Iterator
|
59
|
+
|
60
|
+
TARGET_URL_BASE = "http://localhost:9002/meta/http/sleepy.json"
|
61
|
+
HTTP_REQUEST_OPTIONS = { :connect_timeout => 1.0, :inactivity_timeout => 1.2 }
|
62
|
+
|
63
|
+
attr_reader :requests, :responses
|
64
|
+
|
65
|
+
def initialize env, batch_id, *args
|
66
|
+
@env = env
|
67
|
+
@batch_id = batch_id
|
68
|
+
@requests = []
|
69
|
+
@responses = {:results => {}, :errors => {}}
|
70
|
+
@seen_first_result = false
|
71
|
+
super *args
|
72
|
+
end
|
73
|
+
|
74
|
+
def handle_result req_id, req
|
75
|
+
@responses[:results][req_id] = { :status => req.response_header.http_status, :body => req.response }
|
76
|
+
sep = @seen_first_result ? ",#{SEP}" : ""
|
77
|
+
key = %Q{"#{req_id}":}
|
78
|
+
body = JSON.generate(@responses[:results][req_id])
|
79
|
+
@env.chunked_stream_send([ sep, key, body ].join)
|
80
|
+
end
|
81
|
+
|
82
|
+
def handle_error req_id, req
|
83
|
+
err = req.error.blank? ? 'request error' : req.error
|
84
|
+
@responses[:errors][req_id] = { :error => err }
|
85
|
+
p req.error
|
86
|
+
end
|
87
|
+
|
88
|
+
def perform
|
89
|
+
EM.synchrony do
|
90
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: synchrony block start"
|
91
|
+
|
92
|
+
EM.next_tick{ beg_batch ; beg_results_block }
|
93
|
+
each(
|
94
|
+
proc{|(delay, req_id), iter|
|
95
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: request [#{req_id}, #{delay}]\tconstructing"
|
96
|
+
req = EM::HttpRequest.new(TARGET_URL_BASE, HTTP_REQUEST_OPTIONS).aget(:query => { :delay => delay })
|
97
|
+
|
98
|
+
req.callback do
|
99
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: request [#{req_id}, #{delay}]:\t callback"
|
100
|
+
handle_result(req_id, req)
|
101
|
+
@seen_first_result ||= true
|
102
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: request [#{req_id}, #{delay}]:\t iter.next"
|
103
|
+
iter.next
|
104
|
+
end
|
105
|
+
|
106
|
+
req.errback do
|
107
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: request [#{req_id}, #{delay}]:\t errback"
|
108
|
+
handle_error(req_id, req)
|
109
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: request [#{req_id}, #{delay}]:\t iter.next"
|
110
|
+
iter.next
|
111
|
+
end
|
112
|
+
|
113
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: request [#{req_id}, #{delay}]\tconstructed"
|
114
|
+
|
115
|
+
}, proc{
|
116
|
+
end_results_block
|
117
|
+
@env.chunked_stream_send [",", SEP].join
|
118
|
+
@env.chunked_stream_send JSON.generate({:errors => responses[:errors], :completed_in => (Time.now.utc.to_f - @env[:start_time].to_f)})[1..-2]
|
119
|
+
end_batch
|
120
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: closing stream"
|
121
|
+
@env.chunked_stream_close
|
122
|
+
}
|
123
|
+
)
|
124
|
+
@env.logger.debug "req #{object_id} @#{@batch_id}: synchrony block end"
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
|
129
|
+
SEP = "\n"
|
130
|
+
BEG_BATCH = "{"
|
131
|
+
BEG_RESULTS = %Q<"results":\{>
|
132
|
+
END_RESULTS = "}"
|
133
|
+
END_BATCH = "}"
|
134
|
+
|
135
|
+
def beg_batch
|
136
|
+
@env.chunked_stream_send( [BEG_BATCH, SEP].join )
|
137
|
+
end
|
138
|
+
|
139
|
+
def beg_results_block
|
140
|
+
@env.chunked_stream_send( [BEG_RESULTS, SEP].join )
|
141
|
+
end
|
142
|
+
|
143
|
+
def end_results_block
|
144
|
+
@env.chunked_stream_send [SEP, END_RESULTS].join
|
145
|
+
end
|
146
|
+
|
147
|
+
def end_batch
|
148
|
+
@env.chunked_stream_send( [SEP, END_BATCH].join )
|
149
|
+
end
|
150
|
+
|
151
|
+
end
|
152
|
+
|
153
|
+
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require 'spork'
|
3
|
+
require 'bundler'
|
4
|
+
$:<< '../lib' << 'lib' << 'vendor/goliath/lib'
|
5
|
+
|
6
|
+
Spork.prefork do
|
7
|
+
Bundler.setup
|
8
|
+
Bundler.require
|
9
|
+
|
10
|
+
require 'goliath/test_helper'
|
11
|
+
|
12
|
+
::RSpec.configure do |c|
|
13
|
+
c.include Goliath::TestHelper, :example_group => {
|
14
|
+
:file_path => /spec\/integration/
|
15
|
+
}
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
Spork.each_run do
|
20
|
+
# This code will be run each time you run your specs.
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
|
25
|
+
|
metadata
ADDED
@@ -0,0 +1,203 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: son_of_a_batch
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.2
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Philip (flip) Kromer for Infochimps
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-04-23 00:00:00 -05:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: em-synchrony
|
18
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
19
|
+
none: false
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
type: :runtime
|
25
|
+
prerelease: false
|
26
|
+
version_requirements: *id001
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: em-http-request
|
29
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
30
|
+
none: false
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: "0"
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: *id002
|
38
|
+
- !ruby/object:Gem::Dependency
|
39
|
+
name: yajl-ruby
|
40
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 0.8.2
|
46
|
+
type: :runtime
|
47
|
+
prerelease: false
|
48
|
+
version_requirements: *id003
|
49
|
+
- !ruby/object:Gem::Dependency
|
50
|
+
name: gorillib
|
51
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ~>
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 0.0.4
|
57
|
+
type: :runtime
|
58
|
+
prerelease: false
|
59
|
+
version_requirements: *id004
|
60
|
+
- !ruby/object:Gem::Dependency
|
61
|
+
name: rack-respond_to
|
62
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
63
|
+
none: false
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: "0"
|
68
|
+
type: :runtime
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: *id005
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: rack-abstract-format
|
73
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: "0"
|
79
|
+
type: :runtime
|
80
|
+
prerelease: false
|
81
|
+
version_requirements: *id006
|
82
|
+
- !ruby/object:Gem::Dependency
|
83
|
+
name: bundler
|
84
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
85
|
+
none: false
|
86
|
+
requirements:
|
87
|
+
- - ~>
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 1.0.12
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: *id007
|
93
|
+
- !ruby/object:Gem::Dependency
|
94
|
+
name: yard
|
95
|
+
requirement: &id008 !ruby/object:Gem::Requirement
|
96
|
+
none: false
|
97
|
+
requirements:
|
98
|
+
- - ~>
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: 0.6.7
|
101
|
+
type: :development
|
102
|
+
prerelease: false
|
103
|
+
version_requirements: *id008
|
104
|
+
- !ruby/object:Gem::Dependency
|
105
|
+
name: jeweler
|
106
|
+
requirement: &id009 !ruby/object:Gem::Requirement
|
107
|
+
none: false
|
108
|
+
requirements:
|
109
|
+
- - ~>
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: 1.5.2
|
112
|
+
type: :development
|
113
|
+
prerelease: false
|
114
|
+
version_requirements: *id009
|
115
|
+
- !ruby/object:Gem::Dependency
|
116
|
+
name: rspec
|
117
|
+
requirement: &id010 !ruby/object:Gem::Requirement
|
118
|
+
none: false
|
119
|
+
requirements:
|
120
|
+
- - ~>
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: 2.5.0
|
123
|
+
type: :development
|
124
|
+
prerelease: false
|
125
|
+
version_requirements: *id010
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: rcov
|
128
|
+
requirement: &id011 !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ">="
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: "0"
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: *id011
|
137
|
+
description: Smelt from a plentiferous gallimaufry of requests an agglomerated bale of responses. With, y'know, concurrency and all that.
|
138
|
+
email: coders@infochimps.com
|
139
|
+
executables: []
|
140
|
+
|
141
|
+
extensions: []
|
142
|
+
|
143
|
+
extra_rdoc_files:
|
144
|
+
- LICENSE.txt
|
145
|
+
- README.textile
|
146
|
+
files:
|
147
|
+
- .document
|
148
|
+
- .rspec
|
149
|
+
- Gemfile
|
150
|
+
- Gemfile.lock
|
151
|
+
- LICENSE.txt
|
152
|
+
- README.textile
|
153
|
+
- Rakefile
|
154
|
+
- VERSION
|
155
|
+
- app/endpoints/sleepy.rb
|
156
|
+
- app/views/debug.haml
|
157
|
+
- app/views/layout.haml
|
158
|
+
- app/views/root.haml
|
159
|
+
- config/bootstrap.rb
|
160
|
+
- config/son_of_a_batch-private.example.rb
|
161
|
+
- config/son_of_a_batch-private.rb
|
162
|
+
- config/son_of_a_batch.rb
|
163
|
+
- lib/boot.rb
|
164
|
+
- lib/son_of_a_batch.rb
|
165
|
+
- public/stylesheets/style.css
|
166
|
+
- son_of_a_batch.gemspec
|
167
|
+
- son_of_a_batch.rb
|
168
|
+
- spec/son_of_a_batch_spec.rb
|
169
|
+
- spec/spec_helper.rb
|
170
|
+
has_rdoc: true
|
171
|
+
homepage: http://infochimps.com/labs
|
172
|
+
licenses:
|
173
|
+
- MIT
|
174
|
+
post_install_message:
|
175
|
+
rdoc_options: []
|
176
|
+
|
177
|
+
require_paths:
|
178
|
+
- lib
|
179
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
180
|
+
none: false
|
181
|
+
requirements:
|
182
|
+
- - ">="
|
183
|
+
- !ruby/object:Gem::Version
|
184
|
+
hash: 1754308861720703327
|
185
|
+
segments:
|
186
|
+
- 0
|
187
|
+
version: "0"
|
188
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
189
|
+
none: false
|
190
|
+
requirements:
|
191
|
+
- - ">="
|
192
|
+
- !ruby/object:Gem::Version
|
193
|
+
version: "0"
|
194
|
+
requirements: []
|
195
|
+
|
196
|
+
rubyforge_project:
|
197
|
+
rubygems_version: 1.5.0
|
198
|
+
signing_key:
|
199
|
+
specification_version: 3
|
200
|
+
summary: Smelt from a plentiferous gallimaufry of requests an agglomerated bale of responses. With, y'know, concurrency and all that.
|
201
|
+
test_files:
|
202
|
+
- spec/son_of_a_batch_spec.rb
|
203
|
+
- spec/spec_helper.rb
|