rocketjob 5.0.0.rc1 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c53fb037073ba38fd229b6a0573a2e0a28a1f28bbb0750bc895f2f2d46b57d44
-  data.tar.gz: c5ab8fcbe268aa97909448b4e053cf54698b1f690fe537899f1ed4a5c7f7a1ed
+  metadata.gz: 601a4455f5e51191440ce959b6325545d51fcaf531d179dfcd78a4ec016b44c6
+  data.tar.gz: 7ea8fc1b5cc632461d01e608f5c93cb33e379e14138764fc9e424d15f9c07679
 SHA512:
-  metadata.gz: 69973ec5d30e2d9a63a338119121decb9c20ce8b5d0aba3d9a566799f80eaad626bb2055361bda7cbf5e60085b0fd82c68082f499e151d3a982659457173c5ff
-  data.tar.gz: 8a7b75c3d55fdc1c38c549efc91b8e7cf806ac7aa323a2c0ed1ede9f7e4b7518840d282b172446e5f5f9762685dd5a0639ce9e5fc151d173a59212a0ac5d5ee7
+  metadata.gz: 823280f2250aaa7e2bef010b8b7477f1db1d2f04631c9ce133b80545768e23cab98a14cbc597630988b69b02cfc115acf34a647f50f84fc35e61a708af4c2c09
+  data.tar.gz: e562aa988c120459637cd11108a4401440faa674d1a3226a8c7e77a91ec31635935349803a9c17ace9b3603c14f8a97ec2e8196ba20a79bf518a429807152dce

data/lib/rocket_job/batch/io.rb

@@ -58,14 +58,14 @@ module RocketJob
       #       Searches for the first "\r\n" or "\n" and then uses that as the
       #       delimiter for all subsequent records
       #
-      #   stream_mode: [:line | :row | :record]
+      #   stream_mode: [:line | :array | :hash]
       #     :line
       #       Uploads the file a line (String) at a time for processing by workers.
-      #     :row
+      #     :array
       #       Parses each line from the file as an Array and uploads each array for processing by workers.
-      #     :record
+      #     :hash
       #       Parses each line from the file into a Hash and uploads each hash for processing by workers.
-      #     See IOStreams::Stream#each_line, IOStreams::Stream#each_row, and IOStreams::Stream#each_record.
+      #     See IOStreams::Stream#each.
       #
       # Example:
       #   # Load plain text records from a file
@@ -118,6 +118,11 @@ module RocketJob
       def upload(stream = nil, file_name: nil, category: :main, stream_mode: :line, on_first: nil, **args, &block)
         raise(ArgumentError, 'Either stream, or a block must be supplied') unless stream || block
 
+        stream_mode = stream_mode.to_sym
+        # Backward compatibility with existing v4 jobs
+        stream_mode = :array if stream_mode == :row
+        stream_mode = :hash if stream_mode == :record
+
         count             =
           if block
             input(category).upload(on_first: on_first, &block)
@@ -126,7 +131,7 @@ module RocketJob
             path.file_name        = file_name if file_name
             self.upload_file_name = path.file_name
             input(category).upload(on_first: on_first) do |io|
-              path.public_send("each_#{stream_mode}".to_sym, **args) { |line| io << line }
+              path.each(stream_mode, **args) { |line| io << line }
             end
           end
         self.record_count = (record_count || 0) + count
@@ -336,7 +341,7 @@ module RocketJob
       #   end
       #
       # Example: Add a header and/or trailer record to the downloaded file, letting the line writer add the line breaks:
-      #   IOStreams.path('/tmp/file.txt.gz').line_writer do |writer|
+      #   IOStreams.path('/tmp/file.txt.gz').writer(:line) do |writer|
       #     writer << "Header"
       #     job.download do |line|
       #       writer << line
@@ -352,7 +357,7 @@ module RocketJob
 
         return output(category).download(header_line: header_line, &block) if block
 
-        IOStreams.new(stream).line_writer(**args) do |io|
+        IOStreams.new(stream).writer(:line, **args) do |io|
           output(category).download(header_line: header_line) { |record| io << record }
         end
       end

data/lib/rocket_job/batch/tabular/input.rb

@@ -13,18 +13,18 @@ module RocketJob
           field :tabular_input_header, type: Array, class_attribute: true, user_editable: true
           field :tabular_input_format, type: Symbol, default: :csv, class_attribute: true, user_editable: true
 
-          # tabular_input_mode: [:line | :row | :record]
+          # tabular_input_mode: [:line | :array | :hash]
           #   :line
           #     Uploads the file a line (String) at a time for processing by workers.
-          #   :row
+          #   :array
           #     Parses each line from the file as an Array and uploads each array for processing by workers.
-          #   :record
+          #   :hash
           #     Parses each line from the file into a Hash and uploads each hash for processing by workers.
-          #   See IOStreams#each_line, IOStreams#each_row, and IOStreams#each_record.
+          #   See IOStreams#each.
           field :tabular_input_mode, type: Symbol, default: :line, class_attribute: true, user_editable: true, copy_on_restart: true
 
           validates_inclusion_of :tabular_input_format, in: IOStreams::Tabular.registered_formats
-          validates_inclusion_of :tabular_input_mode, in: %i[line row record]
+          validates_inclusion_of :tabular_input_mode, in: %i[line array hash row record]
           validate :tabular_input_header_present
 
           class_attribute :tabular_input_white_list
@@ -72,16 +72,16 @@ module RocketJob
               tabular_input_cleanse_header
               self.tabular_input_header = tabular_input.header.columns
             end
-            super(input_stream, on_first: parse_header, stream_mode: tabular_input_mode, **args, &block)
-          when :row
+            super(input_stream, on_first: parse_header, stream_mode: :line, **args, &block)
+          when :array, :row
             set_header = -> (row) do
               tabular_input.header.columns = row
               tabular_input_cleanse_header
               self.tabular_input_header = tabular_input.header.columns
             end
-            super(input_stream, on_first: set_header, stream_mode: tabular_input_mode, **args, &block)
-          when :record
-            super(input_stream, stream_mode: tabular_input_mode, **args, &block)
+            super(input_stream, on_first: set_header, stream_mode: :array, **args, &block)
+          when :hash, :record
+            super(input_stream, stream_mode: :hash, **args, &block)
           else
             raise(ArgumentError, "Invalid tabular_input_mode: #{stream_mode.inspect}")
           end
@@ -113,7 +113,7 @@ module RocketJob
         end
 
         def tabular_input_header_present
-          return if tabular_input_header.present? || !tabular_input.header? || (tabular_input_mode == :record)
+          return if tabular_input_header.present? || !tabular_input.header? || (tabular_input_mode == :hash || tabular_input_mode == :record)
 
           errors.add(:tabular_input_header, "is required when tabular_input_format is #{tabular_input_format.inspect}")
         end

data/lib/rocket_job/cli.rb

@@ -10,9 +10,11 @@ module RocketJob
     include SemanticLogger::Loggable
     attr_accessor :environment, :pidfile, :directory, :quiet,
                   :log_level, :log_file, :mongo_config, :symmetric_encryption_config,
-                  :max_workers, :include_filter, :exclude_filter, :where_filter
+                  :max_workers, :include_filter, :exclude_filter, :where_filter, :server,
+                  :stop_server, :kill_server, :pause_server, :resume_server, :thread_dump, :list_servers, :refresh
 
     def initialize(argv)
+      @server                      = true
       @quiet                       = false
       @environment                 = nil
       @pidfile                     = nil
@@ -23,13 +25,19 @@ module RocketJob
       @symmetric_encryption_config = nil
       @include_filter              = nil
       @exclude_filter              = nil
+      @stop_server                 = nil
+      @kill_server                 = nil
+      @pause_server                = nil
+      @resume_server               = nil
+      @thread_dump                 = nil
+      @list_servers                = nil
       parse(argv)
     end
 
     # Run a RocketJob::Server from the command line
     def run
       Thread.current.name = 'rocketjob main'
-      RocketJob.server!
+      RocketJob.server! if server
       setup_environment
       setup_logger
       rails? ? boot_rails : boot_standalone
@@ -39,14 +47,22 @@ module RocketJob
       # In case Rails did not load the Mongoid Config
       RocketJob::Config.load!(environment, mongo_config, symmetric_encryption_config) if ::Mongoid::Config.clients.empty?
 
+      return perform_server_action(stop_server, :stop) if stop_server
+      return perform_server_action(kill_server, :kill) if kill_server
+      return perform_server_action(pause_server, :pause) if pause_server
+      return perform_server_action(resume_server, :resume) if resume_server
+      return perform_server_action(thread_dump, :thread_dump) if thread_dump
+      return perform_list_servers(list_servers) if list_servers
+
       Supervisor.run
     end
 
     def rails?
-      @rails ||= begin
-        boot_file = Pathname.new(directory).join('config/environment.rb').expand_path
-        boot_file.file?
-      end
+      @rails ||=
+        begin
+          boot_file = Pathname.new(directory).join('config/environment.rb').expand_path
+          boot_file.file?
+        end
     end
 
     # Initialize the Rails environment
@@ -155,6 +171,68 @@ module RocketJob
       end
     end
 
+    def perform_list_servers(filter)
+      return list_the_servers(filter) unless refresh
+
+      while true
+        list_the_servers(filter)
+        sleep(refresh)
+        puts
+      end
+    end
+
+    def list_the_servers(filter)
+      format = "%50.50s %20.20s %20.20s %20.20s %10.10s"
+      puts format % %w[Server\ Name Workers(Current/Max) Started Heartbeat State]
+      header = "=" * 50
+      puts format % [header, header, header, header, header]
+      query = filter == :all ? RocketJob::Server.all : RocketJob::Server.where(name: /#{filter}/)
+      query.each do |server|
+        workers   = "#{server&.heartbeat&.workers}/#{server.max_workers}"
+        duration  = Time.now - (server.started_at || Time.now)
+        started   = "#{RocketJob.seconds_as_duration(duration)} ago"
+        duration  = Time.now - (server&.heartbeat&.updated_at || Time.now)
+        heartbeat = "#{RocketJob.seconds_as_duration(duration)} ago"
+        puts format % [server.name, workers, started, heartbeat, server.state]
+      end
+      0
+    end
+
+    def perform_server_action(server_name, action)
+      server_ids(server_name).each { |server_id| RocketJob::Subscribers::Server.publish(action, server_id: server_id) }
+      #RocketJob::Subscribers::Worker.publish(:stop, worker_id: 1, server_id: RocketJob::Server.running.last.id)
+      0
+    end
+
+    # Returns server ids for the supplied exact server name, or partial match.
+    #
+    # When no ':' is supplied a partial hostname lookup is performed.
+    #
+    # Example: Exact server name (hostname and pid) match:
+    #   "9cdbe7e995bc:1"
+    #
+    # Example: Matches all servers that contain the string '.batch.user.org':
+    #   ".batch.user.org"
+    def server_ids(server_name)
+      raise(ArgumentError, "Missing server name") unless server_name
+
+      return [nil] if server_name == :all
+
+      hostname, pid = server_name.split(":")
+      raise(ArgumentError, "Missing server name in: #{server_name}") unless hostname
+
+      if pid
+        server = RocketJob::Server.where(name: server_name).first
+        raise(ArgumentError, "No server with exact name: #{server_name} was found.") unless server
+        return [server.id]
+      end
+
+      server_ids = RocketJob::Server.where(name: /#{hostname}/).collect(&:id)
+      raise(ArgumentError, "No server with partial name: #{server_name} was found.") if server_ids.empty?
+
+      server_ids
+    end
+
     # Parse command line options placing results in the corresponding instance variables
     def parse(argv)
       parser        = OptionParser.new do |o|
@@ -201,6 +279,39 @@ module RocketJob
         o.on('-s', '--symmetric-encryption SYMMETRIC_ENCRYPTION_CONFIG_FILE_NAME', 'Path and filename of Symmetric Encryption config file. Default: config/symmetric-encryption.yml') do |arg|
           @symmetric_encryption_config = arg
         end
+        o.on('--list [FILTER]', "List active servers. Supply either an exact server name or a partial name as a filter.") do |filter|
+          @quiet        = true
+          @server       = false
+          @list_servers = filter || :all
+        end
+        o.on('--refresh [SECONDS]', "When listing active servers, update the list by this number of seconds. Defaults to every 1 second.") do |seconds|
+          @refresh = (seconds || 1).to_s.to_f
+        end
+        o.on('--stop [SERVER_NAME]', "Send event to stop a server once all in-process workers have completed. Optionally supply the complete or partial name of the server(s) to stop. Default: All servers.") do |server_name|
+          @quiet       = true
+          @server      = false
+          @stop_server = server_name || :all
+        end
+        o.on('--kill [SERVER_NAME]', "Send event to hard kill a server. Optionally supply the complete or partial name of the server(s) to kill. Default: All servers.") do |server_name|
+          @quiet       = true
+          @server      = false
+          @kill_server = server_name || :all
+        end
+        o.on('--pause [SERVER_NAME]', "Send event to pause a server. Optionally supply the complete or partial name of the server(s) to pause. Default: All servers.") do |server_name|
+          @quiet        = true
+          @server       = false
+          @pause_server = server_name || :all
+        end
+        o.on('--resume [SERVER_NAME]', "Send event to resume a server. Optionally supply the complete or partial name of the server(s) to resume. Default: All servers.") do |server_name|
+          @quiet         = true
+          @server        = false
+          @resume_server = server_name || :all
+        end
+        o.on('--dump [SERVER_NAME]', "Send event for a server to send a worker thread dump to its log file. Optionally supply the complete or partial name of the server(s). Default: All servers.") do |server_name|
+          @quiet       = true
+          @server      = false
+          @thread_dump = server_name || :all
+        end
         o.on('-v', '--version', 'Print the version information') do
           puts "Rocket Job v#{RocketJob::VERSION}"
           exit 1

data/lib/rocket_job/supervisor.rb

@@ -81,11 +81,11 @@ module RocketJob
           end
         end
 
+        synchronize { server.refresh(worker_pool.living_count) }
+
         self.class.wait_for_event(Config.heartbeat_seconds)
 
         break if self.class.shutdown?
-
-        synchronize { server.refresh(worker_pool.living_count) }
       end
     end
 

data/lib/rocket_job/version.rb

@@ -1,3 +1,3 @@
 module RocketJob
-  VERSION = '5.0.0.rc1'.freeze
+  VERSION = '5.0.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rocketjob
 version: !ruby/object:Gem::Version
-  version: 5.0.0.rc1
+  version: 5.0.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-09 00:00:00.000000000 Z
+date: 2020-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aasm
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta2
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta2
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: mongoid
   requirement: !ruby/object:Gem::Requirement
@@ -192,9 +192,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.0.6
 signing_key: