iron_worker_ng 0.10.2 → 0.10.3

data/lib/iron_worker_ng/cli.rb

@@ -0,0 +1,275 @@
+module IronWorkerNG
+  class CLI
+    LOG_ENTRY = '        '
+    LOG_GROUP = '------> '
+
+    def initialize
+      @client = nil
+
+      @env = nil
+      @project_id = nil
+    end
+
+    def beta?
+      ENV['IRON_BETA'] == '1'
+    end
+
+    def env=(env)
+      @client = nil
+
+      @env = env
+    end
+
+    def project_id=(project_id)
+      @client = nil
+
+      @project_id = project_id
+    end
+
+    def log(msg)
+      IronCore::Logger.info 'IronWorkerNG', msg
+    end
+
+    def log_group(msg)
+      IronCore::Logger.info 'IronWorkerNG', LOG_GROUP + msg
+    end
+
+    def client
+      if @client.nil?
+        log_group "Creating client"
+
+        @client = IronWorkerNG::Client.new(:env => @env, :project_id => @project_id)
+
+        project = client.projects.get
+
+        log "Project '#{project.name}' with id='#{project.id}'"
+      end
+
+      @client
+    end
+
+    def upload(name, params, options)
+      client
+
+      log_group "Creating code package"
+
+      code = IronWorkerNG::Code::Base.new(name)
+
+      code.name(params[:name]) unless params[:name].nil?
+
+      log "Code package name is '#{code.name}'"
+      log "Max concurrency set to '#{options[:max_concurrency]}'" unless options[:max_concurrency].nil?
+      log "Retries set to '#{options[:retries]}'" unless options[:retries].nil?
+      log "Retries delay set to '#{options[:retries_delay]}'" unless options[:retries_delay].nil?
+
+      if code.remote_build_command
+        log_group "Uploading and building code package '#{code.name}'"
+      else
+        log_group "Uploading code package '#{code.name}'"
+      end
+
+      if (params[:async] || params['async']) && code.remote_build_command
+        builder_task_id = client.codes.create_async(code, options)
+
+        log 'Code package is building'
+        log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{builder_task_id}' for more info"
+      else
+        code_id = client.codes.create(code, options).id
+        code_info = client.codes.get(code_id)
+
+        log "Code package uploaded with id='#{code_id}' and revision='#{code_info.rev}'"
+        log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/code/#{code_id}' for more info"
+      end
+    end
+
+    def queue(name, params, options)
+      client
+
+      log_group "Queueing task"
+
+      id = client.tasks.create(name, params[:payload] || params['payload'], options).id
+
+      log "Code package '#{name}' queued with id='#{id}'"
+      log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{id}' for more info"
+    end
+
+    def schedule(name, params, options)
+      client
+
+      log_group "Scheduling task"
+
+      id = client.schedules.create(name, params[:payload] || params['payload'], options).id
+
+      log "Code package '#{name}' scheduled with id='#{id}'"
+      log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/scheduled_jobs/#{id}' for more info"
+    end
+
+    def retry(task_id, params, options)
+      client
+
+      log_group "Retrying task with id='#{task_id}'"
+
+      retry_task_id = client.tasks.retry(task_id, options).id
+
+      log "Task retried with id='#{retry_task_id}'"
+      log "Check 'https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{retry_task_id}' for more info"
+    end
+
+    def getlog(task_id, params, options)
+      client
+
+      live = params[:live] || params['live']
+      wait = params[:wait] || params['wait']
+
+      log_group "Getting log for task with id='#{task_id}'"
+
+      log = ''
+
+      if live
+        begin
+          log = client.tasks.log(task_id)
+        rescue IronCore::Error
+        end
+      else
+        if wait
+          client.tasks.wait_for(task_id)
+        end
+
+        log = client.tasks.log(task_id)
+      end
+
+      print log
+
+      if live
+        client.tasks.wait_for(task_id) do |task|
+          if task.status == 'running'
+            begin
+              next_log = client.tasks.log(task_id)
+              print next_log[log.length .. - 1]
+              log = next_log
+            rescue IronCore::Error
+            end
+          end
+        end
+
+        begin
+          next_log = client.tasks.log(task_id)
+          print next_log[log.length .. - 1]
+        rescue IronCore::Error
+        end
+      end
+    end
+
+    def run(name, params, options)
+      log_group "Creating code package"
+
+      code = IronWorkerNG::Code::Base.new(name)
+
+      log "Code package name is '#{code.name}'"
+
+      log_group "Running '#{code.name}'"
+
+      code.run(params[:payload] || params['payload'])
+    end
+
+    def install(name, params, options)
+      log_group "Installing dependencies for code package with name='#{name}'"
+
+      code = IronWorkerNG::Code::Base.new(name)
+
+      code.install
+    end
+
+    def webhook(name, params, options)
+      client
+
+      log_group 'Generating code package webhook'
+
+      log 'You can invoke your worker by POSTing to the following URL'
+      log "#{client.api.url(client.api.project_id)}/tasks/webhook?code_name=#{name}&oauth=#{client.api.token}"
+    end
+
+    def info_code(name, params, options)
+      client
+
+      log_group 'Getting code package info'
+
+      codes = client.codes.list({:all => true})
+      code = codes.find { |code| code.name == name }
+
+      unless code
+        log_name "Code package with name='#{name}' not found"
+        exit 1
+      end
+
+      data = []
+      data << ['id', code.id]
+      data << ['name', code.name]
+      data << ['revision', code.rev]
+      data << ['uploaded', parse_time(code.latest_change) || '-']
+      data << ['max concurrency', code.max_concurrency || '-']
+      data << ['retries', code.retries || '-']
+      data << ['retries delay', code.retries_delay || '-']
+      data << ['info', "https://hud.iron.io/tq/projects/#{client.api.project_id}/code/#{code.id}"]
+      data << ['tasks info', "https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{code.id}/activity"]
+
+      display_table(data)
+    end
+
+    def info_task(task_id, params, options)
+      client
+
+      log_group 'Getting task info'
+
+      task = client.tasks.get(task_id)
+
+      data = []
+      data << ['id', task.id]
+      data << ['code package', task.code_name]
+      data << ['code revision', task.code_rev]
+      data << ['status', task.status]
+      data << ['priority', task.priority || 2]
+      data << ['queued', parse_time(task.created_at) || '-']
+      data << ['started', parse_time(task.start_time) || '-']
+      data << ['finished', parse_time(task.end_time) || '-']
+      data << ['payload', task.payload]
+      data << ['info', "https://hud.iron.io/tq/projects/#{client.api.project_id}/jobs/#{task.id}"]
+
+      display_table(data)
+    end
+
+    def info_schedule(schedule_id, params, options)
+      client
+
+      log_group 'Getting schedule info'
+
+      schedule = client.schedules.get(schedule_id)
+
+      data = []
+      data << ['id', schedule.id]
+      data << ['code package', schedule.code_name]
+      data << ['status', schedule.status]
+      data << ['created', parse_time(schedule.created_at) || '-']
+      data << ['next start', parse_time(schedule.next_start) || '-']
+      data << ['run count', schedule.run_count || '-']
+      data << ['payload', schedule.payload]
+      data << ['info', "https://hud.iron.io/tq/projects/#{client.api.project_id}/scheduled_jobs/#{schedule.id}"]
+
+      display_table(data)
+    end
+
+    def parse_time(s)
+      t = Time.parse(s)
+
+      return nil if t == Time.utc(1)
+
+      t
+    end
+
+    def display_table(t)
+      t.each do |r|
+        log sprintf('%-16s %s', r[0], r[1])
+      end
+    end
+  end
+end

data/lib/iron_worker_ng/client.rb

@@ -51,7 +51,7 @@ module IronWorkerNG
     def codes_list(options = {})
       IronCore::Logger.debug 'IronWorkerNG', "Calling codes.list with options='#{options.to_s}'"
 
-      all = options.delete(:all) || options.delete('all')
+      all = options[:all] || options['all']
 
       if all
         result = []
@@ -60,9 +60,9 @@ module IronWorkerNG
         per_page = options[:per_page] || options['per_page'] || 100
 
         while true
-          next_codes = codes_list(options.merge({:page => page}))
+          next_codes = codes_list(options.merge({:page => page}).delete_if { |name| name == :all || name == 'all' })
 
-          result += next_codes 
+          result += next_codes
 
           break if next_codes.length != per_page
           page += 1
@@ -83,40 +83,52 @@ module IronWorkerNG
     def codes_create(code, options = {})
       IronCore::Logger.debug 'IronWorkerNG', "Calling codes.create with code='#{code.to_s}' and options='#{options.to_s}'"
 
-      async = options.delete(:async) || options.delete('async')
-
       container_file = code.create_container
 
       if code.remote_build_command.nil?
         res = @api.codes_create(code.name, container_file, 'sh', '__runner__.sh', options)
       else
-        builder_code_name = code.name + (code.name.capitalize == code.name ? '::Builder' : '::builder')
-
-        IronCore::Logger.info 'IronWorkerNG', 'Uploading builder'
+        builder_code_name = code.name + (code.name[0].upcase == code.name[0] ? '::Builder' : '::builder')
 
         @api.codes_create(builder_code_name, container_file, 'sh', '__runner__.sh', options)
 
         builder_task = tasks.create(builder_code_name, :code_name => code.name, :client_options => @api.options.to_json, :codes_create_options => options.to_json)
 
-        if async
-          IronCore::Logger.info 'IronWorkerNG', 'Running builder asynchronously'
+        builder_task = tasks.wait_for(builder_task.id)
+
+        if builder_task.status != 'complete'
+          log = tasks.log(builder_task.id)
 
           File.unlink(container_file)
 
-          return builder_task.id
+          IronCore::Logger.error 'IronWorkerNG', "Error while remote building worker\n" + log, IronCore::Error
         end
 
-        IronCore::Logger.info 'IronWorkerNG', 'Waiting for builder to complete'
+        res = JSON.parse(builder_task.msg)
+      end
 
-        builder_task = tasks.wait_for(builder_task.id)
+      File.unlink(container_file)
 
-        unless builder_task.status == 'complete'
-          log = tasks.log(builder_task.id)
+      OpenStruct.new(res)
+    end
 
-          IronCore::Logger.error 'IronWorkerNG', 'Error while remote building worker: ' + log, IronCore::Error
-        end
+    def codes_create_async(code, options = {})
+      IronCore::Logger.debug 'IronWorkerNG', "Calling codes.create_async with code='#{code.to_s}' and options='#{options.to_s}'"
 
-        res = JSON.parse(builder_task.msg)
+      container_file = code.create_container
+
+      if code.remote_build_command.nil?
+        res = @api.codes_create(code.name, container_file, 'sh', '__runner__.sh', options)
+      else
+        builder_code_name = code.name + (code.name[0].upcase == code.name[0] ? '::Builder' : '::builder')
+
+        @api.codes_create(builder_code_name, container_file, 'sh', '__runner__.sh', options)
+
+        builder_task = tasks.create(builder_code_name, :code_name => code.name, :client_options => @api.options.to_json, :codes_create_options => options.to_json)
+
+        File.unlink(container_file)
+
+        return builder_task.id
       end
 
       File.unlink(container_file)
@@ -202,7 +214,7 @@ module IronWorkerNG
       true
     end
 
-    def tasks_wait_for(task_id, options = {})
+    def tasks_wait_for(task_id, options = {}, &block)
       IronCore::Logger.debug 'IronWorkerNG', "Calling tasks.wait_for with task_id='#{task_id}' and options='#{options.to_s}'"
 
       options[:sleep] ||= options['sleep'] || 5
@@ -210,7 +222,7 @@ module IronWorkerNG
       task = tasks_get(task_id)
 
       while task.status == 'queued' || task.status == 'running'
-        yield task if block_given?
+        block.call(task) unless block.nil?
         sleep options[:sleep]
         task = tasks_get(task_id)
       end

data/lib/iron_worker_ng/code/base.rb

@@ -1,9 +1,10 @@
-require 'tmpdir'
-require 'zip/zip'
 require 'fileutils'
 
-require_relative 'dir_container'
+require_relative 'container/base'
+require_relative 'container/zip'
+require_relative 'container/dir'
 require_relative '../feature/base'
+require_relative '../feature/common/merge_exec'
 require_relative '../feature/common/merge_file'
 require_relative '../feature/common/merge_dir'
 
@@ -38,19 +39,19 @@ module IronWorkerNG
 
         @inside_builder = false
 
-        wfiles = []
+        worker_files = []
 
         if args.length == 1 && args[0].is_a?(String)
-          if args[0].end_with?('.worker') || args[0].end_with?('.workerfile')
-            @name = args[0].gsub(/\.worker$/, '').gsub(/\.workerfile$/, '')
+          if args[0].end_with?('.worker')
+            @name = args[0].gsub(/\.worker$/, '')
           else
             @name = args[0]
           end
         elsif args.length == 1 && args[0].is_a?(Hash)
           @name = args[0][:name] || args[0]['name']
 
-          wfile = args[0][:workerfile] || args[0]['workerfile']
-          wfiles << wfile unless wfile.nil?
+          worker_file = args[0][:workerfile] || args[0]['workerfile']
+          worker_files << worker_file unless worker_file.nil?
 
           exec = args[0][:exec] || args[0]['exec'] || args[0][:worker] || args[0]['worker']
           unless exec.nil?
@@ -63,23 +64,20 @@ module IronWorkerNG
         end
 
         unless @name.nil?
-          wfiles << @name + '.worker'
-          wfiles << @name + '.workerfile'
+          worker_files << @name + '.worker'
         end
 
-        wfiles << 'Workerfile'
+        worker_files.each do |worker_file|
+          IronWorkerNG::Fetcher.fetch(worker_file) do |content|
+            unless content.nil?
+              IronCore::Logger.info 'IronWorkerNG', "Found workerfile with path='#{worker_file}'"
 
-        wfiles.each do |wfile|
-          src, clean = IronWorkerNG::Fetcher.fetch(wfile)
+              eval(content)
 
-          unless src.nil?
-            IronCore::Logger.info 'IronWorkerNG', "Found workerfile with path='#{wfile}'"
+              @base_dir = File.dirname(worker_file) == '.' ? '' : File.dirname(worker_file) + '/'
 
-            eval(src)
-
-            @base_dir = File.dirname(wfile) == '.' ? '' : File.dirname(wfile) + '/'
-
-            break
+              break
+            end
           end
         end
 
@@ -228,33 +226,29 @@ RUNNER
 
         fixate
 
-        container_name = Dir.tmpdir + '/' + Dir::Tmpname.make_tmpname('iron-worker-ng-', "container#{local ? '' : '.zip'}")
+        container = local ? IronWorkerNG::Code::Container::Dir.new : IronWorkerNG::Code::Container::Zip.new
 
-        IronCore::Logger.debug 'IronWorkerNG', "Creating #{local ? 'local ' : ''}code container '#{container_name}'"
+        IronCore::Logger.debug 'IronWorkerNG', "Creating #{local ? 'local ' : ''}code container '#{container.name}'"
 
         if local
-          container = IronWorkerNG::Code::DirContainer.new(container_name)
-
           bundle(container)
         else
           if @remote_build_command
             @dest_dir = '__build__/'
           end
 
-          Zip::ZipFile.open(container_name, Zip::ZipFile::CREATE) do |container|
-            bundle(container)
+          bundle(container)
 
-            if @remote_build_command
-              IronCore::Logger.info 'IronWorkerNG', 'Creating builder'
+          if @remote_build_command
+            IronCore::Logger.info 'IronWorkerNG', 'Creating builder'
               
-              builder = IronWorkerNG::Code::Builder.new
-              builder.remote_build_command = @remote_build_command
+            builder = IronWorkerNG::Code::Builder.new
+            builder.remote_build_command = @remote_build_command
 
-              builder.gem('iron_worker_ng')
-              
-              builder.fixate
-              builder.bundle(container)
-            end
+            builder.gem('iron_worker_ng')
+            builder.fixate
+
+            builder.bundle(container)
           end
           
           if @remote_build_command
@@ -262,7 +256,9 @@ RUNNER
           end
         end
 
-        container_name
+        container.close
+
+        container.name
       end
 
       def run(params = {})
@@ -278,12 +274,24 @@ RUNNER
 
         system("sh #{container_name}/__runner__.sh -d #{container_name} -payload #{container_name}/__payload__ -id 0")
 
-        FileUtils.rm_f(container_name)
+        FileUtils.rm_rf(container_name)
       end
 
       def install
       end
 
+      def workerfile(remote = false)
+        commands = []
+
+        commands << "runtime '#{runtime}'"
+        commands << "name '#{name}'"
+        commands << "build '#{remote_build_command}'"
+
+        commands << @features.map { |f| f.command(remote) }
+
+        commands.compact.join("\n")
+      end
+
       def to_s
         "runtime='#{@runtime}', name='#{@name}', exec='#{@inside_builder || @exec.nil? ? '' : @exec.path}'"
       end