rflow 1.2.0 → 1.3.0a1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b847e06bbad6d47e7cd74314add6f499c99c8bc9
-  data.tar.gz: f707f01ec81e823d8c08711bf747f1a57b6a15a6
+  metadata.gz: 17406ab649fb487a35c56adf29bb50e51e192811
+  data.tar.gz: 8ccdc83a2b260d6431f0db4153dca3f1d93061be
 SHA512:
-  metadata.gz: a5791ba65d956750314d374c00fb21bafcaa152a356331d41134929befaf9488f7519da287b796833430300f5f5aafb892465648ee54a6b8acdc3c1503181427
-  data.tar.gz: bf9ebdb5eac95bd3307f86b54d23b872cd6365fd32db98daa962cd5c109f404c8bf3a9d3ebcc36d5b0c12fb7223b1c8d55b61e62678484fd12f5ca6292ac58e4
+  metadata.gz: d23c815a11716c0e12b7147f8a52f4e4019e5390327fb7810ad55479efdc0fd8a7c7de3b381cd60a5979ebd895c6c1184a8e3b56c95f9ebc06de03a8f9746808
+  data.tar.gz: 9d5ea8d7074d43ed5d0539d8e1e009984d79fcfa9c076ff008252ccecdd8922e7500217d0fc7e530e35c203780f3c33c62e4a6c4983a9ad732e64f7670f4e38c

data/.ruby-version

@@ -1 +1 @@
-ruby-2.3.0
+ruby-2.3.1

data/Gemfile

@@ -3,7 +3,8 @@ source 'https://rubygems.org'
 gemspec
 
 group :development do
-  gem 'guard'
-  gem 'guard-rspec'
-  gem 'listen', '<3.1'
+  if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create("2.2.5")
+    gem 'guard'
+    gem 'guard-rspec'
+  end
 end

data/lib/rflow.rb

@@ -61,13 +61,37 @@ class RFlow
     master.run! # blocks until EventMachine stops
   end
 
+  # Nice pretty wrapper method to help reduce direct dependencies on EM
+  def self.next_tick(pr = nil, &block)
+    EM.next_tick(pr, &block)
+  end
+
+  def self.default_error_callback(error)
+    RFlow.logger.error "Unhandled error on worker thread: #{error.class}: #{error.message}, because: #{error.backtrace}"
+  end
+
+  # Wrapped version of EM.defer that also fixes logging, releases AR
+  # connections, and catches exceptions that would otherwise propagate to the
+  # main thread magically
+  def self.defer(op = nil, callback = nil, errback = nil, &blk)
+    context = RFlow.logger.clone_logging_context
+    EM.defer(nil, callback, errback || method(:default_error_callback)) do
+      begin
+        RFlow.logger.apply_logging_context context
+        (op || blk).call
+      ensure
+        ActiveRecord::Base.connection_pool.release_connection
+      end
+    end
+  end
+
   # This ought to be in EM, but we'll put it here instead of monkey-patching
   def self.next_tick_and_wait
     mutex = Mutex.new
     condition = ConditionVariable.new
 
     mutex.synchronize do # while locked...
-      EM.next_tick do # schedule a job that will...
+      RFlow.next_tick do # schedule a job that will...
         mutex.synchronize do # grab the lock
           begin
             yield # do its thing...

data/lib/rflow/child_process.rb

@@ -4,6 +4,8 @@ class RFlow
   class ChildProcess
     attr_reader :pid, :name
 
+    SIGINFO = 29
+
     def initialize(name, role = name)
       @name = name
       @role = role
@@ -64,15 +66,7 @@ class RFlow
 
     def register_logging_context
       # arrange for child's name to appear in log messages
-      Log4r::NDC.push sprintf("%-#{RFlow.logger.context_width}s", @name)
-    end
-
-    def clone_logging_context
-      Log4r::NDC.clone_stack
-    end
-
-    def apply_logging_context(context)
-      Log4r::NDC.inherit(context)
+      RFlow.logger.add_logging_context sprintf("%-#{RFlow.logger.context_width}s", @name)
     end
 
     def update_process_name
@@ -103,20 +97,24 @@ class RFlow
       trap_signal 'SIGUSR2' do
         RFlow.logger.toggle_log_level
       end
+
+      trap_signal SIGINFO do
+        RFlow.logger.dump_threads
+      end
     end
 
     def unhandle_signals
-      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD', 'SIGUSR1', 'SIGUSR2'].each do |signal|
+      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD', 'SIGUSR1', 'SIGUSR2', SIGINFO].each do |signal|
         Signal.trap signal, 'DEFAULT'
       end
     end
 
     def trap_signal(signal)
       # Log4r and traps don't mix, so we need to put it in another thread
-      context = clone_logging_context
+      context = RFlow.logger.clone_logging_context
       Signal.trap signal do
         Thread.new do
-          apply_logging_context context
+          RFlow.logger.apply_logging_context context
           yield
         end.join
       end

data/lib/rflow/configuration.rb

@@ -159,7 +159,7 @@ class RFlow
       def merge_defaults!
         Setting::DEFAULTS.each do |name, default_value_or_proc|
           value = default_value_or_proc.is_a?(Proc) ? default_value_or_proc.call() : default_value_or_proc
-          setting = Setting.find_or_create_by_name(:name => name, :value => value)
+          setting = Setting.find_or_create_by(:name => name, :value => value)
           unless setting.valid?
             raise RuntimeError, setting.errors.map {|_, msg| msg }.join(', ')
           end

data/lib/rflow/configuration/migrations/20010101000000_create_settings.rb

@@ -4,7 +4,7 @@ class CreateSettings < ActiveRecord::Migration
       t.string :name, :primary => true
       t.text   :value
 
-      t.timestamps
+      t.timestamps null: false
     end
   end
 

data/lib/rflow/configuration/migrations/20010101000001_create_shards.rb

@@ -8,7 +8,7 @@ class CreateShards < ActiveRecord::Migration
       # STI
       t.string :type
 
-      t.timestamps
+      t.timestamps null: false
     end
 
     add_index :shards, :uuid, :unique => true

data/lib/rflow/configuration/migrations/20010101000002_create_components.rb

@@ -10,7 +10,7 @@ class CreateComponents < ActiveRecord::Migration
       # UUID version of belongs_to :shard
       t.string :shard_uuid
 
-      t.timestamps
+      t.timestamps null: false
     end
 
     add_index :components, :uuid, :unique => true

data/lib/rflow/configuration/migrations/20010101000003_create_ports.rb

@@ -10,7 +10,7 @@ class CreatePorts < ActiveRecord::Migration
       # UUID version of belongs_to :component
       t.string :component_uuid
 
-      t.timestamps
+      t.timestamps null: false
     end
 
     add_index :ports, :uuid, :unique => true

data/lib/rflow/configuration/migrations/20010101000004_create_connections.rb

@@ -18,7 +18,7 @@ class CreateConnections < ActiveRecord::Migration
 
       t.text :options
 
-      t.timestamps
+      t.timestamps null: false
     end
 
     add_index :connections, :uuid, :unique => true

data/lib/rflow/configuration/ruby_dsl.rb

@@ -178,13 +178,13 @@ class RFlow
             output_component = RFlow::Configuration::Component.find_by_name spec[:output_component_name]
             raise RFlow::Configuration::Connection::ConnectionInvalid,
               "Component '#{spec[:output_component_name]}' not found at #{spec[:config_line]}" unless output_component
-            output_port = output_component.output_ports.find_or_initialize_by_name :name => spec[:output_port_name]
+            output_port = output_component.output_ports.find_or_initialize_by :name => spec[:output_port_name]
             output_port.save!
 
             input_component = RFlow::Configuration::Component.find_by_name spec[:input_component_name]
             raise RFlow::Configuration::Connection::ConnectionInvalid,
               "Component '#{spec[:input_component_name]}' not found at #{spec[:config_line]}" unless input_component
-            input_port = input_component.input_ports.find_or_initialize_by_name :name => spec[:input_port_name]
+            input_port = input_component.input_ports.find_or_initialize_by :name => spec[:input_port_name]
             input_port.save!
 
             output_shards = output_component.shard.count

data/lib/rflow/configuration/setting.rb

@@ -8,8 +8,6 @@ class RFlow
 
       self.primary_key = 'name'
 
-      attr_accessible :name, :value
-
       DEFAULTS = {
         'rflow.application_name'           => 'rflow',
         'rflow.application_directory_path' => '.',

data/lib/rflow/daemon_process.rb

@@ -2,6 +2,8 @@ require 'rflow/pid_file'
 
 class RFlow
   class DaemonProcess
+    SIGINFO = 29
+
     def initialize(name, role = name, options = {})
       @name = name
       @role = role
@@ -92,16 +94,8 @@ class RFlow
 
     def register_logging_context
       # arrange for process's name to appear in log messages
-      Log4r::NDC.clear
-      Log4r::NDC.push @name
-    end
-
-    def clone_logging_context
-      Log4r::NDC.clone_stack
-    end
-
-    def apply_logging_context(context)
-      Log4r::NDC.inherit(context)
+      RFlow.logger.clear_logging_context
+      RFlow.logger.add_logging_context @name
     end
 
     def update_process_name
@@ -114,7 +108,7 @@ class RFlow
         trap_signal(signal) do |return_code|
           exit_status = if signal == 'SIGCHLD'
                           pid, status = Process.wait2
-                          status.exitstatus
+                          status.exitstatus || 0
                         else
                           0
                         end
@@ -132,10 +126,15 @@ class RFlow
         RFlow.logger.toggle_log_level
         signal_subprocesses 'SIGUSR2'
       end
+
+      trap_signal SIGINFO do
+        RFlow.logger.dump_threads
+        # don't tell child processes to dump, too spammy
+      end
     end
 
     def unhandle_signals
-      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD', 'SIGUSR1', 'SIGUSR2'].each do |signal|
+      ['SIGTERM', 'SIGINT', 'SIGQUIT', 'SIGCHLD', 'SIGUSR1', 'SIGUSR2', SIGINFO].each do |signal|
         Signal.trap signal, 'DEFAULT'
       end
     end
@@ -143,10 +142,10 @@ class RFlow
     def trap_signal(signal)
       # Log4r and traps don't mix, so we need to put it in another thread
       return_code = $?
-      context = clone_logging_context
+      context = RFlow.logger.clone_logging_context
       Signal.trap signal do
         Thread.new do
-          apply_logging_context context
+          RFlow.logger.apply_logging_context context
           yield return_code
         end.join
       end

data/lib/rflow/logger.rb

@@ -66,6 +66,31 @@ class RFlow
       internal_logger.level = LNAMES.index new_log_level
     end
 
+    def dump_threads
+      Thread.list.each do |t|
+        info "Thread #{t.inspect}:"
+        t.backtrace.each {|b| info "  #{b}" }
+        info '---'
+      end
+      info 'Thread dump complete.'
+    end
+
+    def clone_logging_context
+      Log4r::NDC.clone_stack
+    end
+
+    def apply_logging_context(context)
+      Log4r::NDC.inherit(context)
+    end
+
+    def clear_logging_context
+      Log4r::NDC.clear
+    end
+
+    def add_logging_context(context)
+      Log4r::NDC.push context
+    end
+
     private
     def establish_internal_logger
       @internal_logger = Log4r::Logger.new(log_name).tap do |logger|

data/lib/rflow/version.rb

@@ -1,3 +1,3 @@
 class RFlow
-  VERSION = '1.2.0'
+  VERSION = '1.3.0a1'
 end

data/rflow.gemspec

@@ -25,14 +25,14 @@ Gem::Specification.new do |s|
   s.add_dependency 'sys-filesystem', '~> 1.1'
 
   s.add_dependency 'sqlite3', '~> 1.3'
-  s.add_dependency 'activerecord', '~> 3.2'
+  s.add_dependency 'activerecord', '~> 4.0'
 
-  s.add_dependency 'avro', '~> 1.7'
-  s.add_dependency 'em-zeromq', '~> 0.5.0'
+  s.add_dependency 'avro', '~> 1.8'
+  s.add_dependency 'em-zeromq', '~> 0.5'
 
-  s.add_development_dependency 'bundler', '~> 1'
+  s.add_development_dependency 'bundler', '~> 1.0'
   s.add_development_dependency 'rspec', '~> 3.0'
   s.add_development_dependency 'rspec-collection_matchers', '~> 1.0'
   s.add_development_dependency 'rake', '>= 10.3'
-  s.add_development_dependency 'yard', '~> 0.8'
+  s.add_development_dependency 'yard', '~> 0.9'
 end

data/spec/rflow/configuration/ruby_dsl_spec.rb

@@ -156,10 +156,10 @@ class RFlow
         Shard.all.tap do |shards|
           expect(shards.map(&:name)).to eq(['DEFAULT', 's1', 's2', 's3', 's4', 's5'])
           expect(shards.map(&:type)).to eq((['RFlow::Configuration::ProcessShard'] * 4) + (['RFlow::Configuration::ThreadShard'] * 2))
-          expect(shards.first.components.all.map(&:name)).to eq(['first', 'eighth'])
-          expect(shards.second.components.all.map(&:name)).to eq(['second'])
-          expect(shards.third.components.all.map(&:name)).to eq(['third', 'fourth'])
-          expect(shards.fourth.components.all.map(&:name)).to eq(['fifth'])
+          expect(shards.first.components.to_a.map(&:name)).to eq(['first', 'eighth'])
+          expect(shards.second.components.to_a.map(&:name)).to eq(['second'])
+          expect(shards.third.components.to_a.map(&:name)).to eq(['third', 'fourth'])
+          expect(shards.fourth.components.to_a.map(&:name)).to eq(['fifth'])
         end
 
         expect(Port.all.map(&:name)).to eq(['out', 'in', 'out', 'in', 'in2', 'out', 'in', 'in', 'in'])

data/spec/spec_helper.rb

@@ -7,6 +7,9 @@ require 'tmpdir'
 
 I18n.enforce_available_locales = true
 
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+ActiveRecord::Base.logger.level = 4
+
 RSpec.configure do |config|
   config.before(:each) do
     @temp_directory_path = Dir.mktmpdir('rflow')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rflow
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0a1
 platform: ruby
 authors:
 - John Stoneham
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-31 00:00:00.000000000 Z
+date: 2016-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: uuidtools
@@ -73,56 +73,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.2'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: avro
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: em-zeromq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -171,14 +171,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
 description: A Ruby flow-based programming framework that utilizes ZeroMQ for component
   connections and Avro for serialization
 email:
@@ -276,12 +276,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A Ruby flow-based programming framework