iron_worker_ng 0.2.0 → 0.2.1

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.2.1

data/examples/ruby/simple/Gemfile

@@ -0,0 +1,5 @@
+source 'http://rubygems.org'
+
+group :extra do
+  gem 'pry'
+end

data/examples/ruby/simple/sample_worker.rb

@@ -0,0 +1,3 @@
+require 'pry'
+require 'jeweler'
+puts "hello"

data/examples/ruby/simple/simple.rb

@@ -0,0 +1,66 @@
+require 'iron_worker_ng'
+
+# to run examples, you must specify iron.io authentication token and project id
+token, project_id = [ ENV['IRON_IO_TOKEN'], ENV['IRON_IO_PROJECT_ID'] ]
+raise("please set $IRON_IO_TOKEN and $IRON_IO_PROJECT_ID " +
+      "environment variables") unless token and project_id
+
+IronWorkerNG::Logger.logger.level = ::Logger::DEBUG
+
+# initializing api object with them
+client = IronWorkerNG::Client.new(:token => token,
+                                  :project_id => project_id,
+                                  # rest are optinal
+                                  :scheme => 'https',
+                                  :port => 443,
+                                  :api_version => 2,
+                                  :host => 'worker-aws-us-east-1.iron.io')
+
+# creating code bundle
+
+# if not specified, name default to worker name converted from underscore to camel style
+code = IronWorkerNG::Code::Ruby.new
+code.merge_worker('/sample_worker.rb')
+#> code.name == 'SampleWorker'
+
+# still can pass name in constructor
+code = IronWorkerNG::Code::Ruby.new('transmogrify')
+#> code.name == 'transmogrify'
+code.merge_worker(File.dirname(__FILE__) + '/sample_worker.rb')
+
+# once worker merged, following attempts will be ignored
+code.merge_worker('anything')
+#> code.worker.path.end_with? '/worker.rb'
+
+# if worker requires some gems, we 
+
+# we can specify worker dependency on gem
+code.merge_gem('jeweler')
+# or on Gemfile, which is recommended
+code.merge_gemfile(File.dirname(__FILE__) + '/Gemfile',
+                   :default, :extra)
+# all those dependencies will be resolved using bundler gem
+
+# upload code bundle to iron.io
+client.codes_create(code)
+
+# we can retrive code packages list
+codes = client.codes_list
+#> codes.map{|c| c.name}.include?('transmogrify')
+
+code_info = codes.find{|c| c.name == 'transmogrify'}
+# other way to get such info is codes.get:
+same_code_info = client.codes_get(code_info.id)
+#> same_code_info == code_info
+
+# create task to run the bundle
+task_id = client.tasks_create('transmogrify').id
+
+# wait for the task to finish
+client.tasks_wait_for(task_id)
+
+# retriving task log
+log = client.tasks_log(task_id) #> log == "hello\n" -- worker stdout is in log 
+
+# cleanup
+client.codes_delete(code_info.id)

data/lib/iron_worker_ng.rb

@@ -5,3 +5,4 @@ require_relative 'iron_worker_ng/code/base'
 require_relative 'iron_worker_ng/code/ruby'
 require_relative 'iron_worker_ng/code/java'
 require_relative 'iron_worker_ng/code/node'
+require_relative 'iron_worker_ng/code/binary'

data/lib/iron_worker_ng/code/base.rb

@@ -60,23 +60,20 @@ module IronWorkerNG
       end
 
       def create_zip
-        fixate
-
-        init_code = ''
-
-        @features.each do |f|
-          if f.respond_to?(:code_for_init)
-            init_code += f.send(:code_for_init) + "\n"
-          end
+        unless @worker
+          IronWorkerNG::Logger.error 'No worker specified'
+          raise 'No worker specified'
         end
 
+        fixate
+
         zip_name = Dir.tmpdir + '/' + Dir::Tmpname.make_tmpname("iron-worker-ng-", "code.zip")
 
         IronWorkerNG::Logger.debug "Creating code zip '#{zip_name}'"
 
         Zip::ZipFile.open(zip_name, Zip::ZipFile::CREATE) do |zip|
           bundle(zip)
-          create_runner(zip, init_code)
+          create_runner(zip)
         end
 
         zip_name

data/lib/iron_worker_ng/code/binary.rb

@@ -0,0 +1,43 @@
+require_relative '../feature/binary/merge_worker'
+
+module IronWorkerNG
+  module Code
+    class Binary < IronWorkerNG::Code::Base
+      include IronWorkerNG::Feature::Binary::MergeWorker::InstanceMethods
+
+      def create_runner(zip)
+        zip.get_output_stream(runner) do |runner|
+          runner.write <<RUNNER
+#!/bin/sh
+# iron_worker_ng-#{IronWorkerNG.version}
+
+root() {
+  while [ $# -gt 0 ]; do
+    if [ "$1" = "-d" ]; then
+      printf "%s\n" "$2"
+      break
+    fi
+  done
+}
+
+cd "$(root "$@")"
+
+chmod +x #{File.basename(worker.path)}
+
+LD_LIBRARY_PATH=. ./#{File.basename(worker.path)} "$@"
+RUNNER
+        end
+      end
+
+      def runtime
+        'sh'
+      end
+
+      def runner
+        '__runner__.sh'
+      end
+    end
+  end
+end
+
+IronWorkerNG::Code::Base.register_type(:name => 'binary', :klass => IronWorkerNG::Code::Binary)

data/lib/iron_worker_ng/code/java.rb

@@ -7,14 +7,7 @@ module IronWorkerNG
       include IronWorkerNG::Feature::Java::MergeJar::InstanceMethods
       include IronWorkerNG::Feature::Java::MergeWorker::InstanceMethods
 
-      def create_runner(zip, init_code)
-        IronWorkerNG::Logger.info 'Creating java runner'
-
-        unless @worker
-          IronWorkerNG::Logger.error 'No worker specified'
-          raise 'No worker specified'
-        end
-
+      def create_runner(zip)
         classpath_array = []
       
         @features.each do |f|
@@ -27,31 +20,33 @@ module IronWorkerNG
 
         IronWorkerNG::Logger.info "Collected '#{classpath}' classpath"
       
-        zip.get_output_stream('runner.rb') do |runner|
+        zip.get_output_stream(runner) do |runner|
           runner.write <<RUNNER
+#!/bin/sh
 # iron_worker_ng-#{IronWorkerNG.version}
 
-root = nil
-
-($*.length - 2).downto(0) do |i|
-  root = $*[i + 1] if $*[i] == '-d'
-end
-
-Dir.chdir(root)
+root() {
+  while [ $# -gt 0 ]; do
+    if [ "$1" = "-d" ]; then
+      printf "%s\n" "$2"
+      break
+    fi
+  done
+}
 
-#{init_code}
+cd "$(root "$@")"
 
-puts `java -cp #{classpath} #{worker.klass} \#{$*.join(' ')}`
+java -cp #{classpath} #{worker.klass} "$@"
 RUNNER
         end
       end
 
       def runtime
-        'ruby'
+        'sh'
       end
 
       def runner
-        'runner.rb'
+        '__runner__.sh'
       end
     end
   end

data/lib/iron_worker_ng/code/node.rb

@@ -5,39 +5,34 @@ module IronWorkerNG
     class Node < IronWorkerNG::Code::Base
       include IronWorkerNG::Feature::Node::MergeWorker::InstanceMethods
 
-      def create_runner(zip, init_code)
-        IronWorkerNG::Logger.info 'Creating node runner'
-
-        unless @worker
-          IronWorkerNG::Logger.error 'No worker specified'
-          raise 'No worker specified'
-        end
-
-        zip.get_output_stream('runner.rb') do |runner|
+      def create_runner(zip)
+        zip.get_output_stream(runner) do |runner|
           runner.write <<RUNNER
+#!/bin/sh
 # iron_worker_ng-#{IronWorkerNG.version}
 
-root = nil
-
-($*.length - 2).downto(0) do |i|
-  root = $*[i + 1] if $*[i] == '-d'
-end
-
-Dir.chdir(root)
+root() {
+  while [ $# -gt 0 ]; do
+    if [ "$1" = "-d" ]; then
+      printf "%s\n" "$2"
+      break
+    fi
+  done
+}
 
-#{init_code}
+cd "$(root "$@")"
 
-puts `node \#{worker_file_name} \#{$*.join(' ')}`
+node #{File.basename(worker.path)} "$@"
 RUNNER
         end
       end
 
       def runtime
-        'ruby'
+        'sh'
       end
 
       def runner
-        'runner.rb'
+        '__runner__.sh'
       end
     end
   end

data/lib/iron_worker_ng/code/ruby.rb

@@ -9,15 +9,18 @@ module IronWorkerNG
       include IronWorkerNG::Feature::Ruby::MergeGemfile::InstanceMethods
       include IronWorkerNG::Feature::Ruby::MergeWorker::InstanceMethods
 
-      def create_runner(zip, init_code)
-        IronWorkerNG::Logger.info 'Creating ruby runner'
-
-        unless @worker
-          IronWorkerNG::Logger.error 'No worker specified'
-          raise 'No worker specified'
+      def create_runner(zip)
+        gempath_code_array = []
+      
+        @features.each do |f|
+          if f.respond_to?(:code_for_gempath)
+            gempath_code_array << f.send(:code_for_gempath)
+          end
         end
 
-        zip.get_output_stream('runner.rb') do |runner|
+        gempath_code = gempath_code_array.join("\n")
+
+        zip.get_output_stream(runner) do |runner|
           runner.write <<RUNNER
 # iron_worker_ng-#{IronWorkerNG.version}
 
@@ -25,7 +28,7 @@ root = nil
 payload_file = nil
 task_id = nil
 
-($*.length - 2).downto(0) do |i|
+0.upto($*.length - 2) do |i|
   root = $*[i + 1] if $*[i] == '-d'
   payload_file = $*[i + 1] if $*[i] == '-payload'
   task_id = $*[i + 1] if $*[i] == '-id'
@@ -33,7 +36,7 @@ end
 
 Dir.chdir(root)
 
-#{init_code}
+#{gempath_code}
 $:.unshift("\#{root}")
 
 def log(*args)
@@ -81,12 +84,12 @@ def params
   @params
 end
 
-require worker_file_name
+require '#{File.basename(worker.path)}'
 
 worker_class = nil
 
 begin
-  worker_class = Kernel.const_get(worker_class_name)
+  worker_class = Kernel.const_get('#{worker.klass}')
 rescue
 end
 
@@ -112,7 +115,7 @@ RUNNER
       end
 
       def runner
-        'runner.rb'
+        '__runner__.rb'
       end
     end
   end

data/lib/iron_worker_ng/feature/binary/merge_worker.rb

@@ -0,0 +1,50 @@
+module IronWorkerNG
+  module Feature
+    module Binary
+      module MergeWorker
+        class Feature < IronWorkerNG::Feature::Base
+          attr_reader :path
+
+          def initialize(path)
+            @path = File.expand_path(path)
+          end
+
+          def hash_string
+            Digest::MD5.hexdigest(@path + File.mtime(@path).to_i.to_s)
+          end
+
+          def bundle(zip)
+            IronWorkerNG::Logger.debug "Bundling binary worker with path='#{@path}'"
+
+            zip.add(File.basename(@path), @path)
+          end
+        end
+
+        module InstanceMethods
+          attr_reader :worker
+
+          def merge_worker(path)
+            @worker ||= nil 
+
+            unless @worker.nil?
+              IronWorkerNG::Logger.warn "Ignoring attempt to merge binary worker with path='#{path}'"
+              return
+            end
+
+            @name ||= File.basename(path).gsub(/\..*$/, '').capitalize.gsub(/_./) { |x| x[1].upcase }
+
+            @worker = IronWorkerNG::Feature::Binary::MergeWorker::Feature.new(path)
+
+            IronWorkerNG::Logger.info "Merging binary worker with path='#{path}'"
+
+            @features << @worker
+          end
+
+          def self.included(base)
+            IronWorkerNG::Code::Base.register_feature(:name => 'merge_worker', :for_klass => base, :args => 'PATH')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/iron_worker_ng/feature/node/merge_worker.rb

@@ -18,10 +18,6 @@ module IronWorkerNG
 
             zip.add(File.basename(@path), @path)
           end
-
-          def code_for_init
-            "worker_file_name = '#{File.basename(@path)}'"
-          end
         end
 
         module InstanceMethods

data/lib/iron_worker_ng/feature/ruby/merge_gem.rb

@@ -4,6 +4,14 @@ module IronWorkerNG
   module Feature
     module Ruby
       module MergeGem
+        def self.merge_binary?
+          @merge_binary ||= false
+        end
+
+        def self.merge_binary=(merge_binary)
+          @merge_binary = merge_binary
+        end
+
         class Feature < IronWorkerNG::Feature::Base
           attr_reader :spec
 
@@ -31,7 +39,7 @@ module IronWorkerNG
           end
 
           def bundle(zip)
-            if @spec.extensions.length == 0
+            if @spec.extensions.length == 0 || IronWorkerNG::Feature::Ruby::MergeGem.merge_binary?
               IronWorkerNG::Logger.debug "Bundling ruby gem with name='#{@spec.name}' and version='#{@spec.version}'"
 
               zip.add('gems/' + @spec.full_name, gem_path)
@@ -43,7 +51,7 @@ module IronWorkerNG
             end
           end
 
-          def code_for_init
+          def code_for_gempath
             if @spec.extensions.length == 0
               '$:.unshift("#{root}/gems/' + @spec.full_name + '/lib")'
             else

data/lib/iron_worker_ng/feature/ruby/merge_worker.rb

@@ -20,10 +20,6 @@ module IronWorkerNG
 
             zip.add(File.basename(@path), @path)
           end
-
-          def code_for_init
-            "worker_file_name = '#{File.basename(@path)}'\nworker_class_name='#{@klass}'"
-          end
         end
 
         module InstanceMethods

data/test/hello.rb

@@ -0,0 +1 @@
+puts "hello"

data/test/test_basic.rb

@@ -0,0 +1,37 @@
+require './lib/iron_worker_ng.rb'
+require 'test/unit'
+
+class BasicTest < Test::Unit::TestCase
+  attr_accessor :client
+
+  def setup
+    IronWorkerNG::Logger.logger.level = ::Logger::DEBUG
+
+    token, project_id = [ ENV['IRON_IO_TOKEN'], ENV['IRON_IO_PROJECT_ID'] ]
+    raise("please set $IRON_IO_TOKEN and $IRON_IO_PROJECT_ID " +
+          "environment variables") unless token and project_id
+
+    @client = IronWorkerNG::Client.new(:token => token,
+                                       :project_id => project_id )
+  end
+
+  def test_basic
+    code = IronWorkerNG::Code::Ruby.new('test_basic')
+    code.merge_worker(File.dirname(__FILE__) + '/hello.rb')
+    client.codes_create(code)
+    task_id = client.tasks_create('test_basic').id
+    client.tasks_wait_for(task_id)
+    log = client.tasks_log(task_id)
+    assert_equal( "hello\n", log, "worker stdout is in log" )
+  end
+
+  def test_30_codes
+    31.times do |i|
+      code = IronWorkerNG::Code::Ruby.new("test_30_codes_code#{i}")
+      code.merge_worker(File.dirname(__FILE__) + '/hello.rb')
+      client.codes_create(code)
+    end
+    assert_equal( 31, client.codes_list.size )
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iron_worker_ng
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-09 00:00:00.000000000 Z
+date: 2012-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zip
@@ -126,15 +126,20 @@ files:
 - examples/ruby/master_slave/master_slave.rb
 - examples/ruby/master_slave/master_worker.rb
 - examples/ruby/master_slave/slave_worker.rb
+- examples/ruby/simple/Gemfile
+- examples/ruby/simple/sample_worker.rb
+- examples/ruby/simple/simple.rb
 - lib/iron_worker_ng.rb
 - lib/iron_worker_ng/api_client.rb
 - lib/iron_worker_ng/api_client_error.rb
 - lib/iron_worker_ng/client.rb
 - lib/iron_worker_ng/code/base.rb
+- lib/iron_worker_ng/code/binary.rb
 - lib/iron_worker_ng/code/java.rb
 - lib/iron_worker_ng/code/node.rb
 - lib/iron_worker_ng/code/ruby.rb
 - lib/iron_worker_ng/feature/base.rb
+- lib/iron_worker_ng/feature/binary/merge_worker.rb
 - lib/iron_worker_ng/feature/common/merge_dir.rb
 - lib/iron_worker_ng/feature/common/merge_file.rb
 - lib/iron_worker_ng/feature/java/merge_jar.rb
@@ -145,7 +150,9 @@ files:
 - lib/iron_worker_ng/feature/ruby/merge_worker.rb
 - lib/iron_worker_ng/logger.rb
 - lib/iron_worker_ng/version.rb
+- test/hello.rb
 - test/helpers.rb
+- test/test_basic.rb
 homepage: https://github.com/iron-io/iron_worker_ruby_ng
 licenses: []
 post_install_message: 
@@ -160,7 +167,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 862609601
+      hash: 989401361
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: