dynflow 0.8.19 → 0.8.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad55c199553de3dc7ea7869db8600cbda5c78580
-  data.tar.gz: 05149ffe347a46bdecb3744681bba706067c5780
+  metadata.gz: 3b6bf3c0df3b4d89fb0a7bba47fca2bf73c88e25
+  data.tar.gz: 889159760e3a31b5f47f4698049fb0834c403dac
 SHA512:
-  metadata.gz: 1dc99e9b0a52a6c520eee69f48c56d43f5026faf244a89ba2437c42e8d1758855b13df6992b9a1fe27be8725c151773c43d14c2faf940e147519c0a204b3d32c
-  data.tar.gz: aaa0755352ab02b4bea40a4e1e8312f05d2448061d4a6cc29ce23cb8e73abca568a75bdc854caf2fceeeeedaa35c47866da73adb8e2c29716078453848ee0597
+  metadata.gz: 65703698b19511841ab5a33f8433dc096911473e454ae862609b1feaedf0c4f7f400cef69eb21e6b12e1f025efdd65d75a529d32118ca8b201ccc5cf8ae5e2ac
+  data.tar.gz: 3783c92101b70aca94c2fc0d6448374e214d03d900fe4b3b015ea4aab9338dfbb84dc0d71ff1d6b740b8add70533f657f2814c738e1ea6d3dc4bd56f15ad8e8b

data/Gemfile

@@ -3,7 +3,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :concurrent_ruby_ext do
-  gem 'concurrent-ruby-ext', '~> 1.0'
+  gem 'concurrent-ruby-ext', '= 1.0.3'
 end
 
 group :pry do

data/dynflow.gemspec

@@ -29,7 +29,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rack-test"
   s.add_development_dependency "minitest"
   s.add_development_dependency "minitest-reporters"
-  s.add_development_dependency "activerecord"
+  s.add_development_dependency "activerecord", '< 5.0.0'
+  s.add_development_dependency 'activejob', '< 5.0.0'
   s.add_development_dependency "sqlite3"
   s.add_development_dependency "sinatra"
 end

data/lib/dynflow.rb

@@ -49,4 +49,20 @@ module Dynflow
   require 'dynflow/semaphores'
   require 'dynflow/throttle_limiter'
   require 'dynflow/config'
+
+  if defined? ::ActiveJob
+    require 'dynflow/active_job/queue_adapter'
+
+    class Railtie < Rails::Railtie
+      config.before_initialize do
+        ::ActiveJob::QueueAdapters.include(
+          Dynflow::ActiveJob::QueueAdapters
+        )
+      end
+    end
+  end
+
+  if defined? Rails
+    require 'dynflow/rails'
+  end
 end

data/lib/dynflow/active_job/queue_adapter.rb

@@ -0,0 +1,34 @@
+module Dynflow
+  module ActiveJob
+    module QueueAdapters
+      # To use Dynflow, set the queue_adapter config to +:dynflow+.
+      #
+      #   Rails.application.config.active_job.queue_adapter = :dynflow
+      class DynflowAdapter
+        extend ActiveSupport::Concern
+
+        class << self
+          def enqueue(job)
+            ::Rails.application.dynflow.world.trigger(JobWrapper, job.serialize)
+          end
+
+          def enqueue_at(job, timestamp)
+            ::Rails.application.dynflow.world.delay(JobWrapper, { :start_at => Time.at(timestamp) }, job.serialize)
+          end
+        end
+      end
+
+      class JobWrapper < Dynflow::Action
+        def plan(attributes)
+          input[:job_class] = attributes['job_class']
+          input[:job_arguments] = attributes['arguments']
+          plan_self
+        end
+
+        def run
+          input[:job_class].constantize.perform_now(*input[:job_arguments])
+        end
+      end
+    end
+  end
+end

data/lib/dynflow/rails.rb

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+module Dynflow
+  # Class for configuring and preparing the Dynflow runtime environment.
+  class Rails
+    require File.expand_path('../rails/configuration', __FILE__)
+    require File.expand_path('../rails/daemon', __FILE__)
+
+    attr_reader :config
+
+    def initialize(config = Rails::Configuration.new)
+      @required = false
+      @config = config
+    end
+
+    # call this method if your engine uses Dynflow
+    def require!
+      @required = true
+    end
+
+    def required?
+      @required
+    end
+
+    def initialized?
+      !@world.nil?
+    end
+
+    def initialize!
+      return unless @required
+      return @world if @world
+
+      if config.lazy_initialization && defined?(::PhusionPassenger)
+        config.dynflow_logger.
+          warn('Dynflow: lazy loading with PhusionPassenger might lead to unexpected results')
+      end
+      config.initialize_world.tap do |world|
+        @world = world
+
+        unless config.remote?
+          config.run_on_init_hooks(world)
+          # leave this just for long-running executors
+          unless config.rake_task_with_executor?
+            world.auto_execute
+          end
+        end
+      end
+    end
+
+    # Mark that the process is executor. This prevents the remote setting from
+    # applying. Needs to be set up before the world is being initialized
+    def executor!
+      @executor = true
+    end
+
+    def executor?
+      @executor
+    end
+
+    def reinitialize!
+      @world = nil
+      initialize!
+    end
+
+    def world
+      return @world if @world
+
+      initialize! if config.lazy_initialization
+      unless @world
+        raise 'The Dynflow world was not initialized yet. '\
+          'If your plugin uses it, make sure to call Rails.application.dynflow.require! '\
+              'in some initializer'
+      end
+
+      @world
+    end
+
+    attr_writer :world
+
+    def eager_load_actions!
+      config.eager_load_paths.each do |load_path|
+        Dir.glob("#{load_path}/**/*.rb").sort.each do |file|
+          unless loaded_paths.include?(file)
+            require_dependency file
+            loaded_paths << file
+          end
+        end
+      end
+      @world.reload! if @world
+    end
+
+    def loaded_paths
+      @loaded_paths ||= Set.new
+    end
+  end
+end

data/lib/dynflow/rails/configuration.rb

@@ -0,0 +1,146 @@
+module Dynflow
+  class Rails
+    class Configuration
+      # the number of threads in the pool handling the execution
+      attr_accessor :pool_size
+
+      # the size of db connection pool
+      attr_accessor :db_pool_size
+
+      # set true if the executor runs externally (by default true in procution, othewise false)
+      attr_accessor :remote
+      alias remote? remote
+
+      # what transaction adapater should be used, by default, it uses the ActiveRecord
+      # based adapter, expecting ActiveRecord is used as ORM in the application
+      attr_accessor :transaction_adapter
+
+      attr_accessor :eager_load_paths
+
+      attr_accessor :lazy_initialization
+
+      # what rake tasks should run their own executor, not depending on the external one
+      attr_accessor :rake_tasks_with_executor
+
+      def initialize
+        self.pool_size                = 5
+        self.db_pool_size             = pool_size + 5
+        self.remote                   = ::Rails.env.production?
+        self.transaction_adapter      = ::Dynflow::TransactionAdapters::ActiveRecord.new
+        self.eager_load_paths         = []
+        self.lazy_initialization      = !::Rails.env.production?
+        self.rake_tasks_with_executor = %w(db:migrate db:seed)
+
+        @on_init = []
+      end
+
+      # Action related info such as exceptions raised inside the actions' methods
+      # To be overridden in the Rails application
+      def action_logger
+        ::Rails.logger
+      end
+
+      # Dynflow related info about the progress of the execution
+      # To be overridden in the Rails application
+      def dynflow_logger
+        ::Rails.logger
+      end
+
+      def on_init(&block)
+        @on_init << block
+      end
+
+      def run_on_init_hooks(world)
+        @on_init.each { |init| init.call(world) }
+      end
+
+      def initialize_world(world_class = ::Dynflow::World)
+        world_class.new(world_config)
+      end
+
+      # No matter what config.remote says, when the process is marked as executor,
+      # it can't be remote
+      def remote?
+        !::Rails.application.dynflow.executor? &&
+          !rake_task_with_executor? &&
+          @remote
+      end
+
+      def rake_task_with_executor?
+        return false unless defined?(::Rake)
+
+        ::Rake.application.top_level_tasks.any? do |rake_task|
+          rake_tasks_with_executor.include?(rake_task)
+        end
+      end
+
+      def increase_db_pool_size?
+        !::Rails.env.test?
+      end
+
+      # To avoid pottential timeouts on db connection pool, make sure
+      # we have the pool bigger than the thread pool
+      def increase_db_pool_size
+        if increase_db_pool_size?
+          ::ActiveRecord::Base.connection_pool.disconnect!
+
+          config = ::ActiveRecord::Base.configurations[::Rails.env]
+          config['pool'] = db_pool_size if config['pool'].to_i < db_pool_size
+          ::ActiveRecord::Base.establish_connection(config)
+        end
+      end
+
+      # generates the options hash consumable by the Dynflow's world
+      def world_config
+        ::Dynflow::Config.new.tap do |config|
+          config.auto_rescue         = true
+          config.logger_adapter      = ::Dynflow::LoggerAdapters::Delegator.new(action_logger, dynflow_logger)
+          config.pool_size           = 5
+          config.persistence_adapter = initialize_persistence
+          config.transaction_adapter = transaction_adapter
+          config.executor            = ->(world, _) { initialize_executor(world) }
+          config.connector           = ->(world, _) { initialize_connector(world) }
+
+          # we can't do any operation until the Rails.application.dynflow.world is set
+          config.auto_execute        = false
+        end
+      end
+
+      protected
+
+      def default_sequel_adapter_options
+        db_config            = ::ActiveRecord::Base.configurations[::Rails.env].dup
+        db_config['adapter'] = 'postgres' if db_config['adapter'] == 'postgresql'
+        db_config['max_connections'] = db_pool_size if increase_db_pool_size?
+
+        if db_config['adapter'] == 'sqlite3'
+          db_config['adapter'] = 'sqlite'
+          database = db_config['database']
+          unless database == ':memory:'
+            # We need to create separate database for sqlite
+            # to avoid lock conflicts on the database
+            db_config['database'] = "#{File.dirname(database)}/dynflow-#{File.basename(database)}"
+          end
+        end
+        db_config
+      end
+
+      def initialize_executor(world)
+        if remote?
+          false
+        else
+          ::Dynflow::Executors::Parallel.new(world, pool_size)
+        end
+      end
+
+      def initialize_connector(world)
+        ::Dynflow::Connectors::Database.new(world)
+      end
+
+      # Sequel adapter based on Rails app database.yml configuration
+      def initialize_persistence
+        ::Dynflow::PersistenceAdapters::Sequel.new(default_sequel_adapter_options)
+      end
+    end
+  end
+end

data/lib/dynflow/rails/daemon.rb

@@ -0,0 +1,72 @@
+require 'fileutils'
+
+module Dynflow
+  class Rails
+    class Daemon
+      # Load the Rails environment and initialize the executor in this thread.
+      def run(rails_root = Dir.pwd)
+        STDOUT.puts('Starting Rails environment')
+        rails_env_file = File.expand_path('./config/environment.rb', rails_root)
+        unless File.exist?(rails_env_file)
+          raise "#{rails_root} doesn't seem to be a Rails root directory"
+        end
+        ::Rails.application.dynflow.executor!
+        require rails_env_file
+        STDOUT.puts('Everything ready')
+        sleep
+      ensure
+        STDOUT.puts('Exiting')
+      end
+
+      # run the executor as a daemon
+      def run_background(command = 'start', options = {})
+        default_options = { rails_root: Dir.pwd,
+                            process_name: 'dynflow_executor',
+                            pid_dir: File.join(::Rails.root, 'tmp', 'pids'),
+                            log_dir: File.join(::Rails.root, 'log'),
+                            wait_attempts: 300,
+                            wait_sleep: 1,
+                            executors_count: (ENV['EXECUTORS_COUNT'] || 1).to_i }
+        options = default_options.merge(options)
+        FileUtils.mkdir_p(options[:pid_dir])
+        begin
+          require 'daemons'
+        rescue LoadError
+          raise "You need to add gem 'daemons' to your Gemfile if you wish to use it."
+        end
+
+        unless %w(start stop restart run).include?(command)
+          raise "Command exptected to be 'start', 'stop', 'restart', 'run', was #{command.inspect}"
+        end
+
+        STDOUT.puts("Dynflow Executor: #{command} in progress")
+
+        options[:executors_count].times do
+          Daemons.run_proc(options[:process_name],
+                           :multiple => true,
+                           :dir => options[:pid_dir],
+                           :log_dir => options[:log_dir],
+                           :dir_mode => :normal,
+                           :monitor => true,
+                           :log_output => true,
+                           :ARGV => [command]) do |*_args|
+                             begin
+                               ::Logging.reopen
+                               run(options[:rails_root])
+                             rescue => e
+                               STDERR.puts e.message
+                               ::Rails.logger.exception('Failed running Dynflow daemon', e)
+                               exit 1
+                             end
+                           end
+        end
+      end
+
+      protected
+
+      def world
+        ::Rails.application.dynflow.world
+      end
+    end
+  end
+end

data/lib/dynflow/version.rb

@@ -1,3 +1,3 @@
 module Dynflow
-  VERSION = '0.8.19'
+  VERSION = '0.8.20'
 end

data/test/activejob_adapter.rb

@@ -0,0 +1,48 @@
+require_relative 'test_helper'
+require 'active_job'
+require 'dynflow/active_job/queue_adapter'
+
+module Dynflow
+  class SampleJob < ::ActiveJob::Base
+    def perform(msg)
+      puts "This job says #{msg}"
+    end
+  end
+
+  describe 'running jobs' do
+    before(:all) do
+      world = WorldFactory.create_world
+      ::ActiveJob::QueueAdapters.include(::Dynflow::ActiveJob::QueueAdapters)
+      ::ActiveJob::Base.queue_adapter = :dynflow
+      dynflow_mock = Minitest::Mock.new
+      dynflow_mock.expect(:world, world)
+      rails_app_mock = Minitest::Mock.new
+      rails_app_mock .expect(:dynflow, dynflow_mock)
+      rails_mock = Minitest::Mock.new
+      rails_mock.expect(:application, rails_app_mock)
+      ::Rails = rails_mock
+    end
+
+    it 'is able to run the job right away' do
+      out, = capture_subprocess_io do
+        SampleJob.perform_now 'hello'
+      end
+      assert_match(/job says hello/, out)
+    end
+
+    it 'enqueues the job' do
+      out, = capture_subprocess_io do
+        SampleJob.perform_later 'hello'
+      end
+      assert_match(/Enqueued Dynflow::SampleJob/, out)
+    end
+
+    it 'schedules job in the future' do
+      out, = capture_subprocess_io do
+        SampleJob.set(:wait => 1.seconds).perform_later 'hello'
+      end
+      assert_match(/Enqueued Dynflow::SampleJob.*at.*UTC/, out)
+    end
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 0.8.19
+  version: 0.8.20
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-24 00:00:00.000000000 Z
+date: 2017-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -161,16 +161,30 @@ dependencies:
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 5.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 5.0.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -367,6 +381,7 @@ files:
 - lib/dynflow/action/suspended.rb
 - lib/dynflow/action/timeouts.rb
 - lib/dynflow/action/with_sub_plans.rb
+- lib/dynflow/active_job/queue_adapter.rb
 - lib/dynflow/actor.rb
 - lib/dynflow/clock.rb
 - lib/dynflow/config.rb
@@ -444,6 +459,9 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/008_rename_scheduled_plans_to_delayed_plans.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/009_fix_mysql_data_length.rb
+- lib/dynflow/rails.rb
+- lib/dynflow/rails/configuration.rb
+- lib/dynflow/rails/daemon.rb
 - lib/dynflow/round_robin.rb
 - lib/dynflow/semaphores.rb
 - lib/dynflow/semaphores/abstract.rb
@@ -484,6 +502,7 @@ files:
 - lib/dynflow/world.rb
 - test/abnormal_states_recovery_test.rb
 - test/action_test.rb
+- test/activejob_adapter.rb
 - test/clock_test.rb
 - test/concurrency_control_test.rb
 - test/coordinator_test.rb
@@ -561,6 +580,7 @@ summary: DYNamic workFLOW engine
 test_files:
 - test/abnormal_states_recovery_test.rb
 - test/action_test.rb
+- test/activejob_adapter.rb
 - test/clock_test.rb
 - test/concurrency_control_test.rb
 - test/coordinator_test.rb