isono 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -0
- data/Rakefile +38 -0
- data/isono.gemspec +13 -12
- data/lib/isono.rb +5 -4
- data/lib/isono/amqp_client.rb +71 -42
- data/lib/isono/logger.rb +19 -9
- data/lib/isono/manifest.rb +9 -10
- data/lib/isono/node.rb +41 -25
- data/lib/isono/node_modules/base.rb +25 -9
- data/lib/isono/node_modules/event_channel.rb +18 -7
- data/lib/isono/node_modules/job_channel.rb +20 -21
- data/lib/isono/node_modules/job_worker.rb +47 -28
- data/lib/isono/node_modules/rpc_channel.rb +46 -96
- data/lib/isono/rack/job.rb +19 -32
- data/lib/isono/runner/base.rb +150 -0
- data/lib/isono/runner/cli.rb +28 -0
- data/lib/isono/runner/rpc_server.rb +21 -53
- data/lib/isono/thread_pool.rb +24 -19
- data/lib/isono/util.rb +12 -4
- data/lib/isono/version.rb +5 -0
- data/spec/amqp_client_spec.rb +71 -0
- data/spec/event_observable_spec.rb +6 -0
- data/spec/file_channel_spec.rb +263 -0
- data/spec/job_channel_spec.rb +47 -0
- data/spec/logger_spec.rb +45 -0
- data/spec/manifest_spec.rb +43 -0
- data/spec/node_spec.rb +64 -0
- data/spec/resource_loader_spec.rb +113 -0
- data/spec/rpc_channel_spec.rb +172 -0
- data/spec/spec_helper.rb +62 -0
- data/spec/thread_pool_spec.rb +35 -0
- data/spec/util_spec.rb +38 -0
- data/tasks/load_resource_manifest.rake +7 -0
- metadata +79 -43
- data/lib/isono/runner/agent.rb +0 -89
@@ -0,0 +1,71 @@
|
|
1
|
+
|
2
|
+
require File.expand_path('../spec_helper', __FILE__)
|
3
|
+
|
4
|
+
require 'isono'
|
5
|
+
|
6
|
+
class A
|
7
|
+
include Isono::AmqpClient
|
8
|
+
end
|
9
|
+
|
10
|
+
describe Isono::AmqpClient do
|
11
|
+
|
12
|
+
em "connects with default args" do
|
13
|
+
a = A.new
|
14
|
+
a.connect('amqp://localhost/') {
|
15
|
+
a.connected?.should.be.true?
|
16
|
+
a.amqp_client.should.not.nil?
|
17
|
+
EM.next_tick {
|
18
|
+
puts "here"
|
19
|
+
a.close {
|
20
|
+
1.should.equal 1
|
21
|
+
EM.stop
|
22
|
+
}
|
23
|
+
}
|
24
|
+
}
|
25
|
+
end
|
26
|
+
|
27
|
+
em 'run with hook methods' do
|
28
|
+
a = A.new
|
29
|
+
a.instance_eval {
|
30
|
+
@checklist = []
|
31
|
+
def before_connect
|
32
|
+
connected?.should.be.false?
|
33
|
+
checklist << :before_connect
|
34
|
+
end
|
35
|
+
|
36
|
+
def after_connect
|
37
|
+
connected?.should.be.true?
|
38
|
+
checklist << :after_connect
|
39
|
+
end
|
40
|
+
|
41
|
+
def before_close
|
42
|
+
connected?.should.be.true?
|
43
|
+
checklist << :before_close
|
44
|
+
end
|
45
|
+
|
46
|
+
def after_close
|
47
|
+
connected?.should.be.true?
|
48
|
+
checklist << :after_close
|
49
|
+
end
|
50
|
+
|
51
|
+
def checklist
|
52
|
+
@checklist
|
53
|
+
end
|
54
|
+
}
|
55
|
+
a.connect('amqp://localhost/') {
|
56
|
+
EM.next_tick {
|
57
|
+
a.close {
|
58
|
+
a.checklist.member?(:before_connect).should.be.true?
|
59
|
+
a.checklist.member?(:after_connect).should.be.true?
|
60
|
+
a.checklist.member?(:before_close).should.be.true?
|
61
|
+
a.checklist.member?(:after_close).should.be.true?
|
62
|
+
EM.stop
|
63
|
+
}
|
64
|
+
}
|
65
|
+
}
|
66
|
+
|
67
|
+
a.amqp_client.should.not.nil?
|
68
|
+
a.amqp_client.instance_variable_get(:@connection_status).should.is_a?(Proc)
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
@@ -0,0 +1,263 @@
|
|
1
|
+
require File.expand_path('../spec_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'isono'
|
4
|
+
|
5
|
+
require 'fileutils'
|
6
|
+
require 'tempfile'
|
7
|
+
|
8
|
+
MM = Isono::ManagerModules
|
9
|
+
include Isono
|
10
|
+
|
11
|
+
|
12
|
+
def create_sender
|
13
|
+
manifest = Manifest.new(File.expand_path('../', __FILE__)) {
|
14
|
+
node_name :test
|
15
|
+
node_id :id
|
16
|
+
|
17
|
+
manager MM::EventChannel
|
18
|
+
manager MM::FileSenderChannel
|
19
|
+
|
20
|
+
}
|
21
|
+
Agent.new(manifest)
|
22
|
+
end
|
23
|
+
|
24
|
+
def create_receiver
|
25
|
+
manifest = Manifest.new(File.expand_path('../', __FILE__)) {
|
26
|
+
node_name :receiver
|
27
|
+
node_id :xxx
|
28
|
+
|
29
|
+
manager MM::EventChannel
|
30
|
+
manager MM::FileReceiverChannel
|
31
|
+
|
32
|
+
config do |c|
|
33
|
+
c.file_receiver_channel.buffer_dir = '/tmp'
|
34
|
+
c.file_receiver_channel.complete_dir = './complete'
|
35
|
+
end
|
36
|
+
}
|
37
|
+
|
38
|
+
Agent.new(manifest)
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
|
43
|
+
# quick hack for the rspec process fork issue.
|
44
|
+
# each spec runs twice where fork() appers if it's not there.
|
45
|
+
#at_exit { exit! }
|
46
|
+
|
47
|
+
describe "file channel" do
|
48
|
+
|
49
|
+
it "creates new instance" do
|
50
|
+
em_fork(proc {
|
51
|
+
c = create_sender
|
52
|
+
c.connect('amqp://localhost/') {
|
53
|
+
EM.stop
|
54
|
+
}
|
55
|
+
})
|
56
|
+
em_fork(proc {
|
57
|
+
c = create_receiver
|
58
|
+
c.connect('amqp://localhost/') {
|
59
|
+
EM.stop
|
60
|
+
}
|
61
|
+
})
|
62
|
+
|
63
|
+
Process.waitall.all? { |s|
|
64
|
+
s[1].exitstatus == 0
|
65
|
+
}.should.equal true
|
66
|
+
end
|
67
|
+
|
68
|
+
|
69
|
+
def tmp_file_send(sender, to=:receiver, &blk)
|
70
|
+
|
71
|
+
tmpf = Tempfile.open('filechannelspec')
|
72
|
+
tmpf.flush
|
73
|
+
`/usr/bin/shred -s 50K #{tmpf.path}`
|
74
|
+
# manually handles the start_streaming
|
75
|
+
sc = sender.create_sender(to, tmpf.path, File.basename(tmpf.path), :checksum=>true)
|
76
|
+
|
77
|
+
sc.state.should.equal :init
|
78
|
+
sc.src_path.should.equal tmpf.path
|
79
|
+
sc.dst_path.should.equal File.basename(tmpf.path)
|
80
|
+
|
81
|
+
sc.add_observer(:on_read) {
|
82
|
+
}
|
83
|
+
sc.add_observer(:on_eof) {
|
84
|
+
blk.call if blk
|
85
|
+
}
|
86
|
+
|
87
|
+
EM.next_tick {
|
88
|
+
sc.start_streaming
|
89
|
+
}
|
90
|
+
sc
|
91
|
+
end
|
92
|
+
|
93
|
+
it "sends a file" do
|
94
|
+
em_fork(proc {
|
95
|
+
c = create_sender
|
96
|
+
c.connect('amqp://localhost/') {
|
97
|
+
sc = tmp_file_send(MM::FileSenderChannel.instance) {
|
98
|
+
sc.state.should.be.equal :eof
|
99
|
+
EM.stop
|
100
|
+
}
|
101
|
+
}
|
102
|
+
})
|
103
|
+
|
104
|
+
Process.waitall.all? { |s|
|
105
|
+
s[1].exitstatus == 0
|
106
|
+
}.should.equal true
|
107
|
+
end
|
108
|
+
|
109
|
+
it "sends multiple files" do
|
110
|
+
em_fork(proc {
|
111
|
+
c = create_sender
|
112
|
+
c.connect('amqp://localhost/') {
|
113
|
+
sc_lst = []
|
114
|
+
5.times {
|
115
|
+
sc_lst << tmp_file_send(MM::FileSenderChannel.instance)
|
116
|
+
}
|
117
|
+
|
118
|
+
EM.add_periodic_timer(0.5) {
|
119
|
+
EM.stop if sc_lst.all? { |sc| sc.state == :eof }
|
120
|
+
}
|
121
|
+
}
|
122
|
+
})
|
123
|
+
|
124
|
+
Process.waitall.all? { |s|
|
125
|
+
s[1].exitstatus == 0
|
126
|
+
}.should.equal true
|
127
|
+
end
|
128
|
+
|
129
|
+
|
130
|
+
it "send and receive" do
|
131
|
+
em_fork(proc {
|
132
|
+
c = create_receiver
|
133
|
+
flags = {}
|
134
|
+
c.connect('amqp://localhost/') {
|
135
|
+
MM::FileReceiverChannel.instance.add_observer(:on_create_receive_context) { |rc|
|
136
|
+
rc.state.should.equal :open
|
137
|
+
|
138
|
+
flags[:ticket] = rc.ticket
|
139
|
+
rc.add_observer(:on_bof) {
|
140
|
+
flags[:on_bof] = true
|
141
|
+
}
|
142
|
+
rc.add_observer(:on_chunk) {
|
143
|
+
flags[:on_chunk] = true
|
144
|
+
}
|
145
|
+
rc.add_observer(:on_eof) {
|
146
|
+
flags[:on_eof] = true
|
147
|
+
}
|
148
|
+
|
149
|
+
}
|
150
|
+
MM::FileReceiverChannel.instance.add_observer(:on_destroy_receive_context) { |rc|
|
151
|
+
rc.ticket.should.equal flags[:ticket]
|
152
|
+
rc.state.should.equal :close
|
153
|
+
[:on_bof, :on_chunk, :on_eof].each { |k|
|
154
|
+
flags[k].should.true?
|
155
|
+
}
|
156
|
+
EM.stop
|
157
|
+
}
|
158
|
+
|
159
|
+
}
|
160
|
+
})
|
161
|
+
|
162
|
+
|
163
|
+
em_fork(proc {
|
164
|
+
c = create_sender
|
165
|
+
c.connect('amqp://localhost/') {
|
166
|
+
sender = MM::FileSenderChannel.instance
|
167
|
+
tmp_file_send(sender, 'receiver-xxx') {
|
168
|
+
EM.stop
|
169
|
+
}
|
170
|
+
}
|
171
|
+
})
|
172
|
+
|
173
|
+
Process.waitall.all? { |s|
|
174
|
+
s[1].exitstatus == 0
|
175
|
+
}.should.equal true
|
176
|
+
end
|
177
|
+
|
178
|
+
it "send multiple and receive them" do
|
179
|
+
sender_pid = \
|
180
|
+
em_fork(proc {
|
181
|
+
Signal.trap(:TERM) { EM.stop }
|
182
|
+
|
183
|
+
c = create_sender
|
184
|
+
c.connect('amqp://localhost/') {
|
185
|
+
sender = MM::FileSenderChannel.instance
|
186
|
+
5.times {
|
187
|
+
tmp_file_send(sender, 'receiver-xxx')
|
188
|
+
}
|
189
|
+
}
|
190
|
+
})
|
191
|
+
|
192
|
+
em_fork(proc {
|
193
|
+
c = create_receiver
|
194
|
+
flags = {}
|
195
|
+
c.connect('amqp://localhost/') {
|
196
|
+
MM::FileReceiverChannel.instance.add_observer(:on_create_receive_context) { |rc|
|
197
|
+
rc.state.should.equal :open
|
198
|
+
|
199
|
+
flags[rc.ticket] = rc
|
200
|
+
}
|
201
|
+
MM::FileReceiverChannel.instance.add_observer(:on_destroy_receive_context) { |rc|
|
202
|
+
rc.state.should.equal :close
|
203
|
+
flags.has_key?(rc.ticket).should.true?
|
204
|
+
|
205
|
+
if flags.values.all? { |v| v.state == :close }
|
206
|
+
Process.kill(:TERM, sender_pid)
|
207
|
+
EM.stop
|
208
|
+
end
|
209
|
+
}
|
210
|
+
}
|
211
|
+
})
|
212
|
+
|
213
|
+
|
214
|
+
Process.waitall.all? { |s|
|
215
|
+
s[1].exitstatus == 0
|
216
|
+
}.should.equal true
|
217
|
+
end
|
218
|
+
|
219
|
+
|
220
|
+
it "pull a file" do
|
221
|
+
tmpf = Tempfile.open('filechannelspec')
|
222
|
+
tmpf.flush
|
223
|
+
`/usr/bin/shred -s 50K #{tmpf.path}`
|
224
|
+
|
225
|
+
sender_pid = \
|
226
|
+
em_fork(proc {
|
227
|
+
Signal.trap(:TERM) { EM.stop }
|
228
|
+
|
229
|
+
c = create_sender
|
230
|
+
c.connect('amqp://localhost/') {
|
231
|
+
sender = MM::FileSenderChannel.instance
|
232
|
+
sender.add_pull_repository('/tmp/', 'tmp_repos')
|
233
|
+
}
|
234
|
+
})
|
235
|
+
|
236
|
+
em_fork(proc {
|
237
|
+
c = create_receiver
|
238
|
+
flags = {}
|
239
|
+
c.connect('amqp://localhost/') {
|
240
|
+
MM::FileReceiverChannel.instance.add_observer(:on_create_receive_context) { |rc|
|
241
|
+
rc.state.should.equal :open
|
242
|
+
|
243
|
+
flags[rc.ticket] = rc
|
244
|
+
}
|
245
|
+
MM::FileReceiverChannel.instance.add_observer(:on_destroy_receive_context) { |rc|
|
246
|
+
rc.state.should.equal :close
|
247
|
+
flags.has_key?(rc.ticket).should.true?
|
248
|
+
|
249
|
+
if flags.values.all? { |v| v.state == :close }
|
250
|
+
Process.kill(:TERM, sender_pid)
|
251
|
+
EM.stop
|
252
|
+
end
|
253
|
+
}
|
254
|
+
MM::FileReceiverChannel.instance.pull("tmp_repos:#{File.basename(tmpf.path)}")
|
255
|
+
}
|
256
|
+
})
|
257
|
+
|
258
|
+
Process.waitall.all? { |s|
|
259
|
+
s[1].exitstatus == 0
|
260
|
+
}.should.equal true
|
261
|
+
end
|
262
|
+
|
263
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
|
2
|
+
require File.expand_path('../spec_helper', __FILE__)
|
3
|
+
|
4
|
+
require 'isono'
|
5
|
+
include Isono
|
6
|
+
MM=Isono::NodeModules
|
7
|
+
|
8
|
+
def new_node(inst_id, main_cb, pre_cb=nil, post_cb=nil)
|
9
|
+
manifest = Manifest.new(File.expand_path('../', __FILE__)) {
|
10
|
+
node_name 'jobtest'
|
11
|
+
node_instance_id inst_id
|
12
|
+
|
13
|
+
load_module MM::EventChannel
|
14
|
+
load_module MM::RpcChannel
|
15
|
+
load_module MM::JobWorker
|
16
|
+
load_module MM::JobChannel
|
17
|
+
}
|
18
|
+
c = Node.new(manifest)
|
19
|
+
pre_cb.call if pre_cb
|
20
|
+
c.connect('amqp://localhost/') {
|
21
|
+
main_cb.call(c)
|
22
|
+
}
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
describe Isono::NodeModules::JobChannel do
|
27
|
+
|
28
|
+
em "submit a job" do
|
29
|
+
job_id = nil
|
30
|
+
new_node("svr", proc {|c|
|
31
|
+
job = MM::JobChannel.new(c)
|
32
|
+
job.register_endpoint('endpoint1',proc { |req, res|
|
33
|
+
req.command.should.equal 'job1'
|
34
|
+
req.job.job_id.should.equal job_id
|
35
|
+
sleep 1
|
36
|
+
EM.stop
|
37
|
+
})
|
38
|
+
})
|
39
|
+
new_node("cli", proc {|c|
|
40
|
+
job = MM::JobChannel.new(c)
|
41
|
+
job_id = job.submit('endpoint1', 'job1', 1)
|
42
|
+
job_id.should.not.be.nil?
|
43
|
+
})
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
data/spec/logger_spec.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require File.expand_path('../spec_helper', __FILE__)
|
2
|
+
|
3
|
+
require 'isono'
|
4
|
+
|
5
|
+
class LoggerA
|
6
|
+
include Isono::Logger
|
7
|
+
|
8
|
+
class LoggerA1
|
9
|
+
include Isono::Logger
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class LoggerB < LoggerA
|
14
|
+
include Isono::Logger
|
15
|
+
end
|
16
|
+
|
17
|
+
describe "Logger Test" do
|
18
|
+
|
19
|
+
it "call logger method" do
|
20
|
+
a = LoggerA.new
|
21
|
+
a.logger.should_not nil
|
22
|
+
end
|
23
|
+
|
24
|
+
it "call logger instance methods" do
|
25
|
+
l = LoggerA.new
|
26
|
+
l.logger.debug("DEBUG")
|
27
|
+
l.logger.warn("WARN")
|
28
|
+
l.logger.info("INFO")
|
29
|
+
|
30
|
+
l1 = LoggerA::LoggerA1.new
|
31
|
+
l1.logger.debug("DEBUG")
|
32
|
+
|
33
|
+
l1.logger.should.not l.logger
|
34
|
+
end
|
35
|
+
|
36
|
+
it "call logger inherited instance methods" do
|
37
|
+
l = LoggerB.new
|
38
|
+
l.logger.debug("DEBUG")
|
39
|
+
l.logger.warn("WARN")
|
40
|
+
l.logger.info("INFO")
|
41
|
+
|
42
|
+
l.logger.should
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require File.expand_path('../spec_helper', __FILE__)
|
3
|
+
|
4
|
+
require 'isono'
|
5
|
+
include Isono
|
6
|
+
|
7
|
+
class MMStub < Isono::NodeModules::Base
|
8
|
+
config_section do
|
9
|
+
desc "conf1"
|
10
|
+
conf1()
|
11
|
+
desc "conf2"
|
12
|
+
conf2()
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "Isono::Manifest Test" do
|
17
|
+
|
18
|
+
it "define manifest" do
|
19
|
+
m = Isono::Manifest.new('./') {
|
20
|
+
}
|
21
|
+
m.should.is_a? Isono::Manifest
|
22
|
+
m.config.should.is_a? Isono::Manifest::ConfigStruct
|
23
|
+
end
|
24
|
+
|
25
|
+
it "config struct builder" do
|
26
|
+
c = Isono::Manifest::ConfigStruct.new
|
27
|
+
Isono::Manifest::ConfigStructBuilder.new(c).instance_eval { |b|
|
28
|
+
desc "aaa"
|
29
|
+
aaa
|
30
|
+
desc "bbb"
|
31
|
+
bbb 1
|
32
|
+
desc "ccc"
|
33
|
+
b.ccc=2
|
34
|
+
self.should.is_a? Isono::Manifest::ConfigStructBuilder
|
35
|
+
}
|
36
|
+
|
37
|
+
c.desc[:aaa].should.equal 'aaa'
|
38
|
+
c.desc[:bbb].should.equal 'bbb'
|
39
|
+
c.aaa.should.nil?
|
40
|
+
c.bbb.should.equal 1
|
41
|
+
c.ccc.should.equal 2
|
42
|
+
end
|
43
|
+
end
|