iron_worker_ng 0.10.4 → 0.11.0

data/bin/iron_worker

@@ -77,6 +77,12 @@ if command == 'upload'
       params[:async] = true
     end
 
+    if @cli.beta?
+      opts.on('--full-remote-build', 'full remote build') do |v|
+        params[:full_remote_build] = true
+      end
+    end
+
     common_opts(opts)
   end
 

data/lib/iron_worker_ng/cli.rb

@@ -11,7 +11,9 @@ module IronWorkerNG
     end
 
     def beta?
-      ENV['IRON_BETA'] == '1'
+      beta = ENV['IRON_BETA']
+
+      beta == '1' || beta == 1 || beta == 'true' || beta == true || beta == 'beta'
     end
 
     def env=(env)
@@ -57,12 +59,14 @@ module IronWorkerNG
 
       code.name(params[:name]) unless params[:name].nil?
 
+      code.full_remote_build = params[:full_remote_build] unless params[:full_remote_build].nil?
+
       log "Code package name is '#{code.name}'"
       log "Max concurrency set to '#{options[:max_concurrency]}'" unless options[:max_concurrency].nil?
       log "Retries set to '#{options[:retries]}'" unless options[:retries].nil?
       log "Retries delay set to '#{options[:retries_delay]}'" unless options[:retries_delay].nil?
 
-      if code.remote_build_command
+      if code.remote_build_command || code.full_remote_build
         log_group "Uploading and building code package '#{code.name}'"
       else
         log_group "Uploading code package '#{code.name}'"

data/lib/iron_worker_ng/client.rb

@@ -32,6 +32,10 @@ module IronWorkerNG
       end
     end
 
+    def options
+      @api.options
+    end
+
     def token
       @api.token
     end
@@ -85,7 +89,7 @@ module IronWorkerNG
 
       container_file = code.create_container
 
-      if code.remote_build_command.nil?
+      if code.remote_build_command.nil? && (not code.full_remote_build)
         res = @api.codes_create(code.name, container_file, 'sh', '__runner__.sh', options)
       else
         builder_code_name = code.name + (code.name[0].upcase == code.name[0] ? '::Builder' : '::builder')
@@ -117,7 +121,7 @@ module IronWorkerNG
 
       container_file = code.create_container
 
-      if code.remote_build_command.nil?
+      if code.remote_build_command.nil? && (not code.full_remote_build)
         res = @api.codes_create(code.name, container_file, 'sh', '__runner__.sh', options)
       else
         builder_code_name = code.name + (code.name[0].upcase == code.name[0] ? '::Builder' : '::builder')

data/lib/iron_worker_ng/code/base.rb

@@ -17,7 +17,7 @@ module IronWorkerNG
       attr_accessor :base_dir
       attr_accessor :dest_dir
 
-      attr_accessor :inside_builder
+      attr_accessor :full_remote_build
 
       undef exec
       undef gem
@@ -32,13 +32,13 @@ module IronWorkerNG
         @base_dir = ''
         @dest_dir = ''
 
+        @full_remote_build = false
+
         @runtime = nil
 
         @name = nil
         @exec = nil
 
-        @inside_builder = false
-
         worker_files = []
 
         if args.length == 1 && args[0].is_a?(String)
@@ -104,7 +104,7 @@ module IronWorkerNG
       end
 
       def guess_name_for_path(path)
-        File.basename(path).gsub(/#{File.extname(path)}$/, '').capitalize.gsub(/_./) { |x| x[1].upcase }
+        File.basename(path).gsub(/#{File.extname(path)}$/, '')
       end
 
       def name(name = nil)
@@ -174,7 +174,7 @@ module IronWorkerNG
         Digest::MD5.hexdigest(@features.map { |f| f.hash_string }.join)
       end
 
-      def runtime_bundle(container)
+      def runtime_bundle(container, local = false)
       end
 
       def runtime_run_code(local = false)
@@ -186,7 +186,7 @@ module IronWorkerNG
           feature.bundle(container)
         end
 
-        unless @inside_builder
+        if local || ((not remote_build_command) && (not full_remote_build))
           container.get_output_stream(@dest_dir + '__runner__.sh') do |runner|
             runner.write <<RUNNER
 #!/bin/sh
@@ -208,20 +208,18 @@ cd "$(root "$@")"
 #{runtime_run_code(local)}
 RUNNER
           end
-        end
 
-        runtime_bundle(container)
+          runtime_bundle(container, local)
+        end
       end
 
       def create_container(local = false)
-        unless @inside_builder
-          if @exec.nil?
-            IronCore::Logger.error 'IronWorkerNG', 'No exec specified', IronCore::Error
-          end
+        if @exec.nil?
+          IronCore::Logger.error 'IronWorkerNG', 'No exec specified', IronCore::Error
+        end
 
-          if @name.nil?
-            @name = guess_name_for_path(@exec.path)
-          end
+        if @name.nil?
+          @name = guess_name_for_path(@exec.path)
         end
 
         fixate
@@ -231,27 +229,31 @@ RUNNER
         IronCore::Logger.debug 'IronWorkerNG', "Creating #{local ? 'local ' : ''}code container '#{container.name}'"
 
         if local
-          bundle(container)
+          bundle(container, local)
         else
-          if @remote_build_command
+          if @remote_build_command || @full_remote_build
             @dest_dir = '__build__/'
           end
 
-          bundle(container)
+          bundle(container, local)
 
-          if @remote_build_command
+          if @remote_build_command || @full_remote_build
             IronCore::Logger.info 'IronWorkerNG', 'Creating builder'
               
             builder = IronWorkerNG::Code::Builder.new
-            builder.remote_build_command = @remote_build_command
+            builder.builder_remote_build_command = @remote_build_command
 
             builder.gem('iron_worker_ng')
             builder.fixate
 
-            builder.bundle(container)
+            builder.bundle(container, local)
+
+            container.get_output_stream(@name + '.worker') do |wf|
+              wf.write(workerfile)
+            end
           end
           
-          if @remote_build_command
+          if @remote_build_command || @full_remote_build
             @dest_dir = ''
           end
         end
@@ -277,23 +279,22 @@ RUNNER
         FileUtils.rm_rf(container_name)
       end
 
-      def install
+      def install(standalone = false)
       end
 
-      def workerfile(remote = false)
+      def workerfile
         commands = []
 
         commands << "runtime '#{runtime}'"
         commands << "name '#{name}'"
-        commands << "build '#{remote_build_command}'"
 
-        commands << @features.map { |f| f.command(remote) }
+        commands += @features.map { |f| f.command }
 
         commands.compact.join("\n")
       end
 
       def to_s
-        "runtime='#{@runtime}', name='#{@name}', exec='#{@inside_builder || @exec.nil? ? '' : @exec.path}'"
+        "runtime='#{@runtime}', name='#{@name}', exec='#{@exec.path}'"
       end
     end
   end

data/lib/iron_worker_ng/code/builder.rb

@@ -3,6 +3,8 @@ require_relative '../feature/ruby/merge_gem'
 module IronWorkerNG
   module Code
     class Builder < IronWorkerNG::Code::Base
+      attr_accessor :builder_remote_build_command
+
       def initialize(*args, &block)
         @features = []
         @fixators = []
@@ -10,19 +12,24 @@ module IronWorkerNG
         @base_dir = ''
         @dest_dir = ''
 
+        @remote_build_command = nil
+        @full_remote_build = false
+
         runtime(:ruby)
       end
 
-      def bundle(container)
+      def bundle(container, local = false)
         @exec = IronWorkerNG::Feature::Common::MergeExec::Feature.new(self, '__builder__.rb', {})
 
-        super(container)
+        super(container, local)
 
-        container.get_output_stream(@dest_dir + '__builder__.sh') do |builder|
-          builder.write <<BUILDER_SH
+        if builder_remote_build_command
+          container.get_output_stream(@dest_dir + '__builder__.sh') do |builder|
+            builder.write <<BUILDER_SH
 # #{IronWorkerNG.full_version}
-#{remote_build_command}
+#{builder_remote_build_command}
 BUILDER_SH
+          end
         end
 
         container.get_output_stream(@dest_dir + '__builder__.rb') do |builder|
@@ -33,15 +40,23 @@ require 'json'
 
 require 'iron_worker_ng'
 
-exit 1 unless system('cd __build__ && sh ../__builder__.sh && cd ..')
+IronWorkerNG::Feature::Ruby::MergeGem.merge_binary = true
+
+code = IronWorkerNG::Code::Base.new(params[:code_name])
+
+if File.exists?('__builder__.sh')
+  pre_build_list = Dir.glob('__build__/**/**')
 
-::Dir.chdir('__build__')
+  exit 1 unless system('cd __build__ && sh ../__builder__.sh && cd ..')
 
-code = IronWorkerNG::Code::Base.new
-code.inside_builder = true
+  post_build_list = Dir.glob('__build__/**/**')
+
+  (post_build_list.sort - pre_build_list.sort).each do |new_file|
+    code.file(new_file, File.dirname(new_file[10 .. -1])) if File.file?(new_file)
+  end
+end
 
-code.name params[:code_name]
-code.dir '.'
+code.install(true)
 
 client = IronWorkerNG::Client.new(JSON.parse(params[:client_options]))
 

data/lib/iron_worker_ng/code/runtime/java.rb

@@ -10,6 +10,8 @@ module IronWorkerNG
         def runtime_run_code(local = false)
           classpath_array = []
 
+          classpath_array << @exec.path
+
           @features.each do |f|
             if f.respond_to?(:code_for_classpath)
               classpath_array << f.send(:code_for_classpath)

data/lib/iron_worker_ng/code/runtime/php.rb

@@ -4,7 +4,7 @@ module IronWorkerNG
       module PHP
         include IronWorkerNG::Feature::Common::MergeExec::InstanceMethods
 
-        def runtime_bundle(container)
+        def runtime_bundle(container, local = false)
           container.get_output_stream(@dest_dir + '__runner__.php') do |runner|
             runner.write <<PHP_RUNNER
 <?php

data/lib/iron_worker_ng/code/runtime/ruby.rb

@@ -13,7 +13,7 @@ module IronWorkerNG
         include IronWorkerNG::Feature::Ruby::MergeGemDependency::InstanceMethods
         include IronWorkerNG::Feature::Ruby::MergeGemfile::InstanceMethods
 
-        def runtime_bundle(container)
+        def runtime_bundle(container, local = false)
           container.get_output_stream(@dest_dir + '__runner__.rb') do |runner|
             runner.write <<RUBY_RUNNER
 # #{IronWorkerNG.full_version}
@@ -94,12 +94,19 @@ RUBY_RUNNER
 RUN_CODE
         end
 
-        def install
-          gemfile_dir = ::Dir.tmpdir + '/' + ::Dir::Tmpname.make_tmpname('iron-worker-ng-', 'gemfile')
+        def install(standalone = false)
+          gemfile_dir = nil
+          gemfile = nil
 
-          FileUtils.mkdir(gemfile_dir)
+          if standalone
+            gemfile = File.open('Gemfile', 'w')
+          else
+            gemfile_dir = ::Dir.tmpdir + '/' + ::Dir::Tmpname.make_tmpname('iron-worker-ng-', 'gemfile')
 
-          gemfile = File.open(gemfile_dir + '/Gemfile', 'w')
+            FileUtils.mkdir(gemfile_dir)
+
+            gemfile = File.open(gemfile_dir + '/Gemfile', 'w')
+          end
 
           gemfile.puts('source :rubygems')
 
@@ -111,9 +118,11 @@ RUN_CODE
 
           gemfile.close
 
-          puts `cd #{gemfile_dir} && bundle install`
+          puts `cd #{gemfile_dir} && bundle install#{standalone ? ' --standalone' : ''}`
 
-          FileUtils.rm_r(gemfile_dir)
+          unless gemfile_dir.nil?
+            FileUtils.rm_r(gemfile_dir)
+          end
         end
       end
     end

data/lib/iron_worker_ng/feature/base.rb

@@ -6,6 +6,10 @@ module IronWorkerNG
       end
 
       def rebase(path)
+        if IronWorkerNG::Fetcher.remote?(path) || path.start_with?('/')
+          return path
+        end
+
         @code.base_dir + path
       end
 
@@ -32,7 +36,7 @@ module IronWorkerNG
       def bundle(container)
       end
 
-      def command(remote = false)
+      def command
         nil
       end
     end

data/lib/iron_worker_ng/feature/common/merge_dir.rb

@@ -26,15 +26,17 @@ module IronWorkerNG
           def bundle(container)
             IronCore::Logger.debug 'IronWorkerNG', "Bundling dir with path='#{@path}' and dest='#{@dest}'"
 
-            container_add(container, @dest + File.basename(@path), rebase(@path))
+            if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path)))
+              container_add(container, @dest + File.basename(@path), rebase(@path))
+            end
           end
 
-          def command(remote = false)
-            if remote
-              if IronWorkerNG::Fetcher.remote?(rebase(@path))
+          def command
+            if @code.remote_build_command || @code.full_remote_build
+              if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path))
                 "dir '#{rebase(@path)}', '#{@dest}'"
               else
-                "dir '#{@dest}#{File.basename(@path)}', '#{@dest}'"
+                "dir '#{@code.dest_dir}#{@dest}#{File.basename(@path)}', '#{@dest}'"
               end
             else
               "dir '#{@path}', '#{@dest}'"

data/lib/iron_worker_ng/feature/common/merge_exec.rb

@@ -30,15 +30,17 @@ module IronWorkerNG
           def bundle(container)
             IronCore::Logger.debug 'IronWorkerNG', "Bundling exec with path='#{@path}' and args='#{@args.inspect}'"
 
-            container_add(container, File.basename(@path), rebase(@path))
+            if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path)))
+              container_add(container, File.basename(@path), rebase(@path))
+            end
           end
 
-          def command(remote = false)
-            if remote
-              if IronWorkerNG::Fetcher.remote?(rebase(@path))
+          def command
+            if @code.remote_build_command || @code.full_remote_build
+              if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path))
                 "exec '#{rebase(@path)}', #{@args.inspect}"
               else
-                "exec '#{File.basename(@path)}', #{@args.inspect}"
+                "exec '#{@code.dest_dir}#{File.basename(@path)}', #{@args.inspect}"
               end
             else
               "exec '#{@path}', #{@args.inspect}"

data/lib/iron_worker_ng/feature/common/merge_file.rb

@@ -20,15 +20,17 @@ module IronWorkerNG
           def bundle(container)
             IronCore::Logger.debug 'IronWorkerNG', "Bundling file with path='#{@path}' and dest='#{@dest}'"
 
-            container_add(container, @dest + File.basename(@path), rebase(@path))
+            if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path)))
+              container_add(container, @dest + File.basename(@path), rebase(@path))
+            end
           end
 
-          def command(remote = false)
-            if remote
-              if IronWorkerNG::Fetcher.remote?(rebase(@path))
+          def command
+            if @code.remote_build_command || @code.full_remote_build
+              if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path))
                 "file '#{rebase(@path)}', '#{@dest}'"
               else
-                "file '#{@dest}#{File.basename(@path)}', '#{@dest}'"
+                "file '#{@code.dest_dir}#{@dest}#{File.basename(@path)}', '#{@dest}'"
               end
             else
               "file '#{@path}', '#{@dest}'"

data/lib/iron_worker_ng/feature/java/merge_jar.rb

@@ -18,15 +18,17 @@ module IronWorkerNG
           def bundle(container)
             IronCore::Logger.debug 'IronWorkerNG', "Bundling java jar with path='#{@path}'"
 
-            container_add(container, File.basename(@path), rebase(@path))
+            if (not @code.full_remote_build) || (not IronWorkerNG::Fetcher.remote?(rebase(@path)))
+              container_add(container, File.basename(@path), rebase(@path))
+            end
           end
 
-          def command(remote = false)
-            if remote
-              if IronWorkerNG::Fetcher.remote?(rebase(@path))
+          def command
+            if @code.remote_build_command || @code.full_remote_build
+              if @code.full_remote_build && IronWorkerNG::Fetcher.remote?(rebase(@path))
                 "jar '#{rebase(@path)}'"
               else
-                "jar '#{File.basename(@path)}'"
+                "jar '#{@code.dest_dir}#{File.basename(@path)}'"
               end
             else
               "jar '#{@path}'"

data/lib/iron_worker_ng/feature/ruby/merge_gem.rb

@@ -36,22 +36,24 @@ module IronWorkerNG
           end
 
           def bundle(container)
-            if @spec.extensions.length == 0 || IronWorkerNG::Feature::Ruby::MergeGem.merge_binary?
-              IronCore::Logger.debug 'IronWorkerNG', "Bundling ruby gem with name='#{@spec.name}' and version='#{@spec.version}'"
-
-              if File.exists? @spec.full_gem_path
-                container_add(container, '__gems__/gems/' + @spec.full_name, gem_path)
-              else # local gem
-                from_dir = File.dirname @spec.loaded_from
-                @spec.files.each do |fname|
-                  to = '__gems__/gems/' + @spec.full_name + '/' + fname
-                  from = from_dir + '/' + fname
-                  container_add(container, to, from) if File.file? from
+            if not @code.full_remote_build
+              if @spec.extensions.length == 0 || IronWorkerNG::Feature::Ruby::MergeGem.merge_binary?
+                IronCore::Logger.debug 'IronWorkerNG', "Bundling ruby gem with name='#{@spec.name}' and version='#{@spec.version}'"
+
+                if File.exists?(@spec.full_gem_path)
+                  container_add(container, '__gems__/gems/' + @spec.full_name, gem_path)
+                else
+                  from_dir = File.dirname(@spec.loaded_from)
+
+                  @spec.files.each do |fname|
+                    container_add(container, '__gems__/gems/' + @spec.full_name + '/' + fname, from_dir + '/' + fname) if File.file?(from_dir + '/' + fname)
+                  end
                 end
+
+                container_add(container, "__gems__/specifications/#{@spec.full_name}.gemspec", @spec.loaded_from)
+              else
+                IronCore::Logger.warn 'IronWorkerNG', "Skipping ruby gem with name='#{@spec.name}' and version='#{@spec.version}' as it contains native extensions"
               end
-              container_add(container, "__gems__/specifications/#{@spec.full_name}.gemspec", @spec.loaded_from)
-            else
-              IronCore::Logger.warn 'IronWorkerNG', "Skipping ruby gem with name='#{@spec.name}' and version='#{@spec.version}' as it contains native extensions"
             end
           end
         end

data/lib/iron_worker_ng/feature/ruby/merge_gem_dependency.rb

@@ -19,8 +19,12 @@ module IronWorkerNG
             Digest::MD5.hexdigest(@name + @version)
           end
 
-          def command(remote = false)
-            "gem '#{@name}', '#{@version}'"
+          def command
+            if @code.full_remote_build
+              "gem '#{@name}', '#{@version}'"
+            else
+              nil
+            end
           end
         end
 
@@ -38,34 +42,36 @@ module IronWorkerNG
           alias :gem :merge_gem
 
           def merge_gem_dependency_fixate
-            IronCore::Logger.info 'IronWorkerNG', 'Fixating gems dependencies'
+            if not full_remote_build
+              IronCore::Logger.info 'IronWorkerNG', 'Fixating gems dependencies'
 
-            @features.reject! { |f| f.class == IronWorkerNG::Feature::Ruby::MergeGem::Feature }
+              @features.reject! { |f| f.class == IronWorkerNG::Feature::Ruby::MergeGem::Feature }
 
-            deps = @features.reject { |f| f.class != IronWorkerNG::Feature::Ruby::MergeGemDependency::Feature }
+              deps = @features.reject { |f| f.class != IronWorkerNG::Feature::Ruby::MergeGemDependency::Feature }
 
-            if deps.length > 0
-              deps = deps.map { |dep| Bundler::DepProxy.new(Bundler::Dependency.new(dep.name, dep.version.split(', ')), Gem::Platform::RUBY) }
+              if deps.length > 0
+                deps = deps.map { |dep| Bundler::DepProxy.new(Bundler::Dependency.new(dep.name, dep.version.split(', ')), Gem::Platform::RUBY) }
 
-              source = nil
+                source = nil
 
-              begin
-                source = Bundler::Source::Rubygems.new
-              rescue Bundler::GemfileNotFound
-                ENV['BUNDLE_GEMFILE'] = 'Gemfile'
-                source = Bundler::Source::Rubygems.new
-              end
+                begin
+                  source = Bundler::Source::Rubygems.new
+                rescue Bundler::GemfileNotFound
+                  ENV['BUNDLE_GEMFILE'] = 'Gemfile'
+                  source = Bundler::Source::Rubygems.new
+                end
 
-              index = Bundler::Index.build { |index| index.use source.specs }
+                index = Bundler::Index.build { |index| index.use source.specs }
 
-              spec_set = Bundler::Resolver.resolve(deps, index)
+                spec_set = Bundler::Resolver.resolve(deps, index)
 
-              spec_set.to_a.each do |spec|
-                spec.__materialize__
+                spec_set.to_a.each do |spec|
+                  spec.__materialize__
 
-                IronCore::Logger.info 'IronWorkerNG', "Merging ruby gem with name='#{spec.name}' and version='#{spec.version}'"
+                  IronCore::Logger.info 'IronWorkerNG', "Merging ruby gem with name='#{spec.name}' and version='#{spec.version}'"
 
-                @features << IronWorkerNG::Feature::Ruby::MergeGem::Feature.new(self, spec)
+                  @features << IronWorkerNG::Feature::Ruby::MergeGem::Feature.new(self, spec)
+                end
               end
             end
           end

data/lib/iron_worker_ng/version.rb

@@ -1,5 +1,5 @@
 module IronWorkerNG
-  VERSION = "0.10.4"
+  VERSION = "0.11.0"
 
   def self.version
     VERSION

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iron_worker_ng
 version: !ruby/object:Gem::Version
-  version: 0.10.4
+  version: 0.11.0
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-06 00:00:00.000000000 Z
+date: 2012-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: iron_core