iron_worker_ng 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/examples/ruby/simple/Gemfile +5 -0
- data/examples/ruby/simple/sample_worker.rb +3 -0
- data/examples/ruby/simple/simple.rb +66 -0
- data/lib/iron_worker_ng.rb +1 -0
- data/lib/iron_worker_ng/code/base.rb +6 -9
- data/lib/iron_worker_ng/code/binary.rb +43 -0
- data/lib/iron_worker_ng/code/java.rb +15 -20
- data/lib/iron_worker_ng/code/node.rb +15 -20
- data/lib/iron_worker_ng/code/ruby.rb +15 -12
- data/lib/iron_worker_ng/feature/binary/merge_worker.rb +50 -0
- data/lib/iron_worker_ng/feature/node/merge_worker.rb +0 -4
- data/lib/iron_worker_ng/feature/ruby/merge_gem.rb +10 -2
- data/lib/iron_worker_ng/feature/ruby/merge_worker.rb +0 -4
- data/test/hello.rb +1 -0
- data/test/test_basic.rb +37 -0
- metadata +10 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.1
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'iron_worker_ng'
|
2
|
+
|
3
|
+
# to run examples, you must specify iron.io authentication token and project id
|
4
|
+
token, project_id = [ ENV['IRON_IO_TOKEN'], ENV['IRON_IO_PROJECT_ID'] ]
|
5
|
+
raise("please set $IRON_IO_TOKEN and $IRON_IO_PROJECT_ID " +
|
6
|
+
"environment variables") unless token and project_id
|
7
|
+
|
8
|
+
IronWorkerNG::Logger.logger.level = ::Logger::DEBUG
|
9
|
+
|
10
|
+
# initializing api object with them
|
11
|
+
client = IronWorkerNG::Client.new(:token => token,
|
12
|
+
:project_id => project_id,
|
13
|
+
# rest are optinal
|
14
|
+
:scheme => 'https',
|
15
|
+
:port => 443,
|
16
|
+
:api_version => 2,
|
17
|
+
:host => 'worker-aws-us-east-1.iron.io')
|
18
|
+
|
19
|
+
# creating code bundle
|
20
|
+
|
21
|
+
# if not specified, name default to worker name converted from underscore to camel style
|
22
|
+
code = IronWorkerNG::Code::Ruby.new
|
23
|
+
code.merge_worker('/sample_worker.rb')
|
24
|
+
#> code.name == 'SampleWorker'
|
25
|
+
|
26
|
+
# still can pass name in constructor
|
27
|
+
code = IronWorkerNG::Code::Ruby.new('transmogrify')
|
28
|
+
#> code.name == 'transmogrify'
|
29
|
+
code.merge_worker(File.dirname(__FILE__) + '/sample_worker.rb')
|
30
|
+
|
31
|
+
# once worker merged, following attempts will be ignored
|
32
|
+
code.merge_worker('anything')
|
33
|
+
#> code.worker.path.end_with? '/worker.rb'
|
34
|
+
|
35
|
+
# if worker requires some gems, we
|
36
|
+
|
37
|
+
# we can specify worker dependency on gem
|
38
|
+
code.merge_gem('jeweler')
|
39
|
+
# or on Gemfile, which is recommended
|
40
|
+
code.merge_gemfile(File.dirname(__FILE__) + '/Gemfile',
|
41
|
+
:default, :extra)
|
42
|
+
# all those dependencies will be resolved using bundler gem
|
43
|
+
|
44
|
+
# upload code bundle to iron.io
|
45
|
+
client.codes_create(code)
|
46
|
+
|
47
|
+
# we can retrive code packages list
|
48
|
+
codes = client.codes_list
|
49
|
+
#> codes.map{|c| c.name}.include?('transmogrify')
|
50
|
+
|
51
|
+
code_info = codes.find{|c| c.name == 'transmogrify'}
|
52
|
+
# other way to get such info is codes.get:
|
53
|
+
same_code_info = client.codes_get(code_info.id)
|
54
|
+
#> same_code_info == code_info
|
55
|
+
|
56
|
+
# create task to run the bundle
|
57
|
+
task_id = client.tasks_create('transmogrify').id
|
58
|
+
|
59
|
+
# wait for the task to finish
|
60
|
+
client.tasks_wait_for(task_id)
|
61
|
+
|
62
|
+
# retriving task log
|
63
|
+
log = client.tasks_log(task_id) #> log == "hello\n" -- worker stdout is in log
|
64
|
+
|
65
|
+
# cleanup
|
66
|
+
client.codes_delete(code_info.id)
|
data/lib/iron_worker_ng.rb
CHANGED
@@ -60,23 +60,20 @@ module IronWorkerNG
|
|
60
60
|
end
|
61
61
|
|
62
62
|
def create_zip
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
@features.each do |f|
|
68
|
-
if f.respond_to?(:code_for_init)
|
69
|
-
init_code += f.send(:code_for_init) + "\n"
|
70
|
-
end
|
63
|
+
unless @worker
|
64
|
+
IronWorkerNG::Logger.error 'No worker specified'
|
65
|
+
raise 'No worker specified'
|
71
66
|
end
|
72
67
|
|
68
|
+
fixate
|
69
|
+
|
73
70
|
zip_name = Dir.tmpdir + '/' + Dir::Tmpname.make_tmpname("iron-worker-ng-", "code.zip")
|
74
71
|
|
75
72
|
IronWorkerNG::Logger.debug "Creating code zip '#{zip_name}'"
|
76
73
|
|
77
74
|
Zip::ZipFile.open(zip_name, Zip::ZipFile::CREATE) do |zip|
|
78
75
|
bundle(zip)
|
79
|
-
create_runner(zip
|
76
|
+
create_runner(zip)
|
80
77
|
end
|
81
78
|
|
82
79
|
zip_name
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require_relative '../feature/binary/merge_worker'
|
2
|
+
|
3
|
+
module IronWorkerNG
|
4
|
+
module Code
|
5
|
+
class Binary < IronWorkerNG::Code::Base
|
6
|
+
include IronWorkerNG::Feature::Binary::MergeWorker::InstanceMethods
|
7
|
+
|
8
|
+
def create_runner(zip)
|
9
|
+
zip.get_output_stream(runner) do |runner|
|
10
|
+
runner.write <<RUNNER
|
11
|
+
#!/bin/sh
|
12
|
+
# iron_worker_ng-#{IronWorkerNG.version}
|
13
|
+
|
14
|
+
root() {
|
15
|
+
while [ $# -gt 0 ]; do
|
16
|
+
if [ "$1" = "-d" ]; then
|
17
|
+
printf "%s\n" "$2"
|
18
|
+
break
|
19
|
+
fi
|
20
|
+
done
|
21
|
+
}
|
22
|
+
|
23
|
+
cd "$(root "$@")"
|
24
|
+
|
25
|
+
chmod +x #{File.basename(worker.path)}
|
26
|
+
|
27
|
+
LD_LIBRARY_PATH=. ./#{File.basename(worker.path)} "$@"
|
28
|
+
RUNNER
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def runtime
|
33
|
+
'sh'
|
34
|
+
end
|
35
|
+
|
36
|
+
def runner
|
37
|
+
'__runner__.sh'
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
IronWorkerNG::Code::Base.register_type(:name => 'binary', :klass => IronWorkerNG::Code::Binary)
|
@@ -7,14 +7,7 @@ module IronWorkerNG
|
|
7
7
|
include IronWorkerNG::Feature::Java::MergeJar::InstanceMethods
|
8
8
|
include IronWorkerNG::Feature::Java::MergeWorker::InstanceMethods
|
9
9
|
|
10
|
-
def create_runner(zip
|
11
|
-
IronWorkerNG::Logger.info 'Creating java runner'
|
12
|
-
|
13
|
-
unless @worker
|
14
|
-
IronWorkerNG::Logger.error 'No worker specified'
|
15
|
-
raise 'No worker specified'
|
16
|
-
end
|
17
|
-
|
10
|
+
def create_runner(zip)
|
18
11
|
classpath_array = []
|
19
12
|
|
20
13
|
@features.each do |f|
|
@@ -27,31 +20,33 @@ module IronWorkerNG
|
|
27
20
|
|
28
21
|
IronWorkerNG::Logger.info "Collected '#{classpath}' classpath"
|
29
22
|
|
30
|
-
zip.get_output_stream(
|
23
|
+
zip.get_output_stream(runner) do |runner|
|
31
24
|
runner.write <<RUNNER
|
25
|
+
#!/bin/sh
|
32
26
|
# iron_worker_ng-#{IronWorkerNG.version}
|
33
27
|
|
34
|
-
root
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
28
|
+
root() {
|
29
|
+
while [ $# -gt 0 ]; do
|
30
|
+
if [ "$1" = "-d" ]; then
|
31
|
+
printf "%s\n" "$2"
|
32
|
+
break
|
33
|
+
fi
|
34
|
+
done
|
35
|
+
}
|
41
36
|
|
42
|
-
|
37
|
+
cd "$(root "$@")"
|
43
38
|
|
44
|
-
|
39
|
+
java -cp #{classpath} #{worker.klass} "$@"
|
45
40
|
RUNNER
|
46
41
|
end
|
47
42
|
end
|
48
43
|
|
49
44
|
def runtime
|
50
|
-
'
|
45
|
+
'sh'
|
51
46
|
end
|
52
47
|
|
53
48
|
def runner
|
54
|
-
'
|
49
|
+
'__runner__.sh'
|
55
50
|
end
|
56
51
|
end
|
57
52
|
end
|
@@ -5,39 +5,34 @@ module IronWorkerNG
|
|
5
5
|
class Node < IronWorkerNG::Code::Base
|
6
6
|
include IronWorkerNG::Feature::Node::MergeWorker::InstanceMethods
|
7
7
|
|
8
|
-
def create_runner(zip
|
9
|
-
|
10
|
-
|
11
|
-
unless @worker
|
12
|
-
IronWorkerNG::Logger.error 'No worker specified'
|
13
|
-
raise 'No worker specified'
|
14
|
-
end
|
15
|
-
|
16
|
-
zip.get_output_stream('runner.rb') do |runner|
|
8
|
+
def create_runner(zip)
|
9
|
+
zip.get_output_stream(runner) do |runner|
|
17
10
|
runner.write <<RUNNER
|
11
|
+
#!/bin/sh
|
18
12
|
# iron_worker_ng-#{IronWorkerNG.version}
|
19
13
|
|
20
|
-
root
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
14
|
+
root() {
|
15
|
+
while [ $# -gt 0 ]; do
|
16
|
+
if [ "$1" = "-d" ]; then
|
17
|
+
printf "%s\n" "$2"
|
18
|
+
break
|
19
|
+
fi
|
20
|
+
done
|
21
|
+
}
|
27
22
|
|
28
|
-
|
23
|
+
cd "$(root "$@")"
|
29
24
|
|
30
|
-
|
25
|
+
node #{File.basename(worker.path)} "$@"
|
31
26
|
RUNNER
|
32
27
|
end
|
33
28
|
end
|
34
29
|
|
35
30
|
def runtime
|
36
|
-
'
|
31
|
+
'sh'
|
37
32
|
end
|
38
33
|
|
39
34
|
def runner
|
40
|
-
'
|
35
|
+
'__runner__.sh'
|
41
36
|
end
|
42
37
|
end
|
43
38
|
end
|
@@ -9,15 +9,18 @@ module IronWorkerNG
|
|
9
9
|
include IronWorkerNG::Feature::Ruby::MergeGemfile::InstanceMethods
|
10
10
|
include IronWorkerNG::Feature::Ruby::MergeWorker::InstanceMethods
|
11
11
|
|
12
|
-
def create_runner(zip
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
12
|
+
def create_runner(zip)
|
13
|
+
gempath_code_array = []
|
14
|
+
|
15
|
+
@features.each do |f|
|
16
|
+
if f.respond_to?(:code_for_gempath)
|
17
|
+
gempath_code_array << f.send(:code_for_gempath)
|
18
|
+
end
|
18
19
|
end
|
19
20
|
|
20
|
-
|
21
|
+
gempath_code = gempath_code_array.join("\n")
|
22
|
+
|
23
|
+
zip.get_output_stream(runner) do |runner|
|
21
24
|
runner.write <<RUNNER
|
22
25
|
# iron_worker_ng-#{IronWorkerNG.version}
|
23
26
|
|
@@ -25,7 +28,7 @@ root = nil
|
|
25
28
|
payload_file = nil
|
26
29
|
task_id = nil
|
27
30
|
|
28
|
-
($*.length - 2)
|
31
|
+
0.upto($*.length - 2) do |i|
|
29
32
|
root = $*[i + 1] if $*[i] == '-d'
|
30
33
|
payload_file = $*[i + 1] if $*[i] == '-payload'
|
31
34
|
task_id = $*[i + 1] if $*[i] == '-id'
|
@@ -33,7 +36,7 @@ end
|
|
33
36
|
|
34
37
|
Dir.chdir(root)
|
35
38
|
|
36
|
-
#{
|
39
|
+
#{gempath_code}
|
37
40
|
$:.unshift("\#{root}")
|
38
41
|
|
39
42
|
def log(*args)
|
@@ -81,12 +84,12 @@ def params
|
|
81
84
|
@params
|
82
85
|
end
|
83
86
|
|
84
|
-
require
|
87
|
+
require '#{File.basename(worker.path)}'
|
85
88
|
|
86
89
|
worker_class = nil
|
87
90
|
|
88
91
|
begin
|
89
|
-
worker_class = Kernel.const_get(
|
92
|
+
worker_class = Kernel.const_get('#{worker.klass}')
|
90
93
|
rescue
|
91
94
|
end
|
92
95
|
|
@@ -112,7 +115,7 @@ RUNNER
|
|
112
115
|
end
|
113
116
|
|
114
117
|
def runner
|
115
|
-
'
|
118
|
+
'__runner__.rb'
|
116
119
|
end
|
117
120
|
end
|
118
121
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module IronWorkerNG
|
2
|
+
module Feature
|
3
|
+
module Binary
|
4
|
+
module MergeWorker
|
5
|
+
class Feature < IronWorkerNG::Feature::Base
|
6
|
+
attr_reader :path
|
7
|
+
|
8
|
+
def initialize(path)
|
9
|
+
@path = File.expand_path(path)
|
10
|
+
end
|
11
|
+
|
12
|
+
def hash_string
|
13
|
+
Digest::MD5.hexdigest(@path + File.mtime(@path).to_i.to_s)
|
14
|
+
end
|
15
|
+
|
16
|
+
def bundle(zip)
|
17
|
+
IronWorkerNG::Logger.debug "Bundling binary worker with path='#{@path}'"
|
18
|
+
|
19
|
+
zip.add(File.basename(@path), @path)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
module InstanceMethods
|
24
|
+
attr_reader :worker
|
25
|
+
|
26
|
+
def merge_worker(path)
|
27
|
+
@worker ||= nil
|
28
|
+
|
29
|
+
unless @worker.nil?
|
30
|
+
IronWorkerNG::Logger.warn "Ignoring attempt to merge binary worker with path='#{path}'"
|
31
|
+
return
|
32
|
+
end
|
33
|
+
|
34
|
+
@name ||= File.basename(path).gsub(/\..*$/, '').capitalize.gsub(/_./) { |x| x[1].upcase }
|
35
|
+
|
36
|
+
@worker = IronWorkerNG::Feature::Binary::MergeWorker::Feature.new(path)
|
37
|
+
|
38
|
+
IronWorkerNG::Logger.info "Merging binary worker with path='#{path}'"
|
39
|
+
|
40
|
+
@features << @worker
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.included(base)
|
44
|
+
IronWorkerNG::Code::Base.register_feature(:name => 'merge_worker', :for_klass => base, :args => 'PATH')
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -4,6 +4,14 @@ module IronWorkerNG
|
|
4
4
|
module Feature
|
5
5
|
module Ruby
|
6
6
|
module MergeGem
|
7
|
+
def self.merge_binary?
|
8
|
+
@merge_binary ||= false
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.merge_binary=(merge_binary)
|
12
|
+
@merge_binary = merge_binary
|
13
|
+
end
|
14
|
+
|
7
15
|
class Feature < IronWorkerNG::Feature::Base
|
8
16
|
attr_reader :spec
|
9
17
|
|
@@ -31,7 +39,7 @@ module IronWorkerNG
|
|
31
39
|
end
|
32
40
|
|
33
41
|
def bundle(zip)
|
34
|
-
if @spec.extensions.length == 0
|
42
|
+
if @spec.extensions.length == 0 || IronWorkerNG::Feature::Ruby::MergeGem.merge_binary?
|
35
43
|
IronWorkerNG::Logger.debug "Bundling ruby gem with name='#{@spec.name}' and version='#{@spec.version}'"
|
36
44
|
|
37
45
|
zip.add('gems/' + @spec.full_name, gem_path)
|
@@ -43,7 +51,7 @@ module IronWorkerNG
|
|
43
51
|
end
|
44
52
|
end
|
45
53
|
|
46
|
-
def
|
54
|
+
def code_for_gempath
|
47
55
|
if @spec.extensions.length == 0
|
48
56
|
'$:.unshift("#{root}/gems/' + @spec.full_name + '/lib")'
|
49
57
|
else
|
data/test/hello.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
puts "hello"
|
data/test/test_basic.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require './lib/iron_worker_ng.rb'
|
2
|
+
require 'test/unit'
|
3
|
+
|
4
|
+
class BasicTest < Test::Unit::TestCase
|
5
|
+
attr_accessor :client
|
6
|
+
|
7
|
+
def setup
|
8
|
+
IronWorkerNG::Logger.logger.level = ::Logger::DEBUG
|
9
|
+
|
10
|
+
token, project_id = [ ENV['IRON_IO_TOKEN'], ENV['IRON_IO_PROJECT_ID'] ]
|
11
|
+
raise("please set $IRON_IO_TOKEN and $IRON_IO_PROJECT_ID " +
|
12
|
+
"environment variables") unless token and project_id
|
13
|
+
|
14
|
+
@client = IronWorkerNG::Client.new(:token => token,
|
15
|
+
:project_id => project_id )
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_basic
|
19
|
+
code = IronWorkerNG::Code::Ruby.new('test_basic')
|
20
|
+
code.merge_worker(File.dirname(__FILE__) + '/hello.rb')
|
21
|
+
client.codes_create(code)
|
22
|
+
task_id = client.tasks_create('test_basic').id
|
23
|
+
client.tasks_wait_for(task_id)
|
24
|
+
log = client.tasks_log(task_id)
|
25
|
+
assert_equal( "hello\n", log, "worker stdout is in log" )
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_30_codes
|
29
|
+
31.times do |i|
|
30
|
+
code = IronWorkerNG::Code::Ruby.new("test_30_codes_code#{i}")
|
31
|
+
code.merge_worker(File.dirname(__FILE__) + '/hello.rb')
|
32
|
+
client.codes_create(code)
|
33
|
+
end
|
34
|
+
assert_equal( 31, client.codes_list.size )
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: iron_worker_ng
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2012-04-
|
13
|
+
date: 2012-04-15 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: zip
|
@@ -126,15 +126,20 @@ files:
|
|
126
126
|
- examples/ruby/master_slave/master_slave.rb
|
127
127
|
- examples/ruby/master_slave/master_worker.rb
|
128
128
|
- examples/ruby/master_slave/slave_worker.rb
|
129
|
+
- examples/ruby/simple/Gemfile
|
130
|
+
- examples/ruby/simple/sample_worker.rb
|
131
|
+
- examples/ruby/simple/simple.rb
|
129
132
|
- lib/iron_worker_ng.rb
|
130
133
|
- lib/iron_worker_ng/api_client.rb
|
131
134
|
- lib/iron_worker_ng/api_client_error.rb
|
132
135
|
- lib/iron_worker_ng/client.rb
|
133
136
|
- lib/iron_worker_ng/code/base.rb
|
137
|
+
- lib/iron_worker_ng/code/binary.rb
|
134
138
|
- lib/iron_worker_ng/code/java.rb
|
135
139
|
- lib/iron_worker_ng/code/node.rb
|
136
140
|
- lib/iron_worker_ng/code/ruby.rb
|
137
141
|
- lib/iron_worker_ng/feature/base.rb
|
142
|
+
- lib/iron_worker_ng/feature/binary/merge_worker.rb
|
138
143
|
- lib/iron_worker_ng/feature/common/merge_dir.rb
|
139
144
|
- lib/iron_worker_ng/feature/common/merge_file.rb
|
140
145
|
- lib/iron_worker_ng/feature/java/merge_jar.rb
|
@@ -145,7 +150,9 @@ files:
|
|
145
150
|
- lib/iron_worker_ng/feature/ruby/merge_worker.rb
|
146
151
|
- lib/iron_worker_ng/logger.rb
|
147
152
|
- lib/iron_worker_ng/version.rb
|
153
|
+
- test/hello.rb
|
148
154
|
- test/helpers.rb
|
155
|
+
- test/test_basic.rb
|
149
156
|
homepage: https://github.com/iron-io/iron_worker_ruby_ng
|
150
157
|
licenses: []
|
151
158
|
post_install_message:
|
@@ -160,7 +167,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
160
167
|
version: '0'
|
161
168
|
segments:
|
162
169
|
- 0
|
163
|
-
hash:
|
170
|
+
hash: 989401361
|
164
171
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
172
|
none: false
|
166
173
|
requirements:
|