dynflow 0.8.19 → 0.8.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ad55c199553de3dc7ea7869db8600cbda5c78580
4
- data.tar.gz: 05149ffe347a46bdecb3744681bba706067c5780
3
+ metadata.gz: 3b6bf3c0df3b4d89fb0a7bba47fca2bf73c88e25
4
+ data.tar.gz: 889159760e3a31b5f47f4698049fb0834c403dac
5
5
  SHA512:
6
- metadata.gz: 1dc99e9b0a52a6c520eee69f48c56d43f5026faf244a89ba2437c42e8d1758855b13df6992b9a1fe27be8725c151773c43d14c2faf940e147519c0a204b3d32c
7
- data.tar.gz: aaa0755352ab02b4bea40a4e1e8312f05d2448061d4a6cc29ce23cb8e73abca568a75bdc854caf2fceeeeedaa35c47866da73adb8e2c29716078453848ee0597
6
+ metadata.gz: 65703698b19511841ab5a33f8433dc096911473e454ae862609b1feaedf0c4f7f400cef69eb21e6b12e1f025efdd65d75a529d32118ca8b201ccc5cf8ae5e2ac
7
+ data.tar.gz: 3783c92101b70aca94c2fc0d6448374e214d03d900fe4b3b015ea4aab9338dfbb84dc0d71ff1d6b740b8add70533f657f2814c738e1ea6d3dc4bd56f15ad8e8b
data/Gemfile CHANGED
@@ -3,7 +3,7 @@ source 'https://rubygems.org'
3
3
  gemspec
4
4
 
5
5
  group :concurrent_ruby_ext do
6
- gem 'concurrent-ruby-ext', '~> 1.0'
6
+ gem 'concurrent-ruby-ext', '= 1.0.3'
7
7
  end
8
8
 
9
9
  group :pry do
@@ -29,7 +29,8 @@ Gem::Specification.new do |s|
29
29
  s.add_development_dependency "rack-test"
30
30
  s.add_development_dependency "minitest"
31
31
  s.add_development_dependency "minitest-reporters"
32
- s.add_development_dependency "activerecord"
32
+ s.add_development_dependency "activerecord", '< 5.0.0'
33
+ s.add_development_dependency 'activejob', '< 5.0.0'
33
34
  s.add_development_dependency "sqlite3"
34
35
  s.add_development_dependency "sinatra"
35
36
  end
@@ -49,4 +49,20 @@ module Dynflow
49
49
  require 'dynflow/semaphores'
50
50
  require 'dynflow/throttle_limiter'
51
51
  require 'dynflow/config'
52
+
53
+ if defined? ::ActiveJob
54
+ require 'dynflow/active_job/queue_adapter'
55
+
56
+ class Railtie < Rails::Railtie
57
+ config.before_initialize do
58
+ ::ActiveJob::QueueAdapters.include(
59
+ Dynflow::ActiveJob::QueueAdapters
60
+ )
61
+ end
62
+ end
63
+ end
64
+
65
+ if defined? Rails
66
+ require 'dynflow/rails'
67
+ end
52
68
  end
@@ -0,0 +1,34 @@
1
+ module Dynflow
2
+ module ActiveJob
3
+ module QueueAdapters
4
+ # To use Dynflow, set the queue_adapter config to +:dynflow+.
5
+ #
6
+ # Rails.application.config.active_job.queue_adapter = :dynflow
7
+ class DynflowAdapter
8
+ extend ActiveSupport::Concern
9
+
10
+ class << self
11
+ def enqueue(job)
12
+ ::Rails.application.dynflow.world.trigger(JobWrapper, job.serialize)
13
+ end
14
+
15
+ def enqueue_at(job, timestamp)
16
+ ::Rails.application.dynflow.world.delay(JobWrapper, { :start_at => Time.at(timestamp) }, job.serialize)
17
+ end
18
+ end
19
+ end
20
+
21
+ class JobWrapper < Dynflow::Action
22
+ def plan(attributes)
23
+ input[:job_class] = attributes['job_class']
24
+ input[:job_arguments] = attributes['arguments']
25
+ plan_self
26
+ end
27
+
28
+ def run
29
+ input[:job_class].constantize.perform_now(*input[:job_arguments])
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,95 @@
1
+ # -*- coding: utf-8 -*-
2
+ module Dynflow
3
+ # Class for configuring and preparing the Dynflow runtime environment.
4
+ class Rails
5
+ require File.expand_path('../rails/configuration', __FILE__)
6
+ require File.expand_path('../rails/daemon', __FILE__)
7
+
8
+ attr_reader :config
9
+
10
+ def initialize(config = Rails::Configuration.new)
11
+ @required = false
12
+ @config = config
13
+ end
14
+
15
+ # call this method if your engine uses Dynflow
16
+ def require!
17
+ @required = true
18
+ end
19
+
20
+ def required?
21
+ @required
22
+ end
23
+
24
+ def initialized?
25
+ !@world.nil?
26
+ end
27
+
28
+ def initialize!
29
+ return unless @required
30
+ return @world if @world
31
+
32
+ if config.lazy_initialization && defined?(::PhusionPassenger)
33
+ config.dynflow_logger.
34
+ warn('Dynflow: lazy loading with PhusionPassenger might lead to unexpected results')
35
+ end
36
+ config.initialize_world.tap do |world|
37
+ @world = world
38
+
39
+ unless config.remote?
40
+ config.run_on_init_hooks(world)
41
+ # leave this just for long-running executors
42
+ unless config.rake_task_with_executor?
43
+ world.auto_execute
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ # Mark that the process is executor. This prevents the remote setting from
50
+ # applying. Needs to be set up before the world is being initialized
51
+ def executor!
52
+ @executor = true
53
+ end
54
+
55
+ def executor?
56
+ @executor
57
+ end
58
+
59
+ def reinitialize!
60
+ @world = nil
61
+ initialize!
62
+ end
63
+
64
+ def world
65
+ return @world if @world
66
+
67
+ initialize! if config.lazy_initialization
68
+ unless @world
69
+ raise 'The Dynflow world was not initialized yet. '\
70
+ 'If your plugin uses it, make sure to call Rails.application.dynflow.require! '\
71
+ 'in some initializer'
72
+ end
73
+
74
+ @world
75
+ end
76
+
77
+ attr_writer :world
78
+
79
+ def eager_load_actions!
80
+ config.eager_load_paths.each do |load_path|
81
+ Dir.glob("#{load_path}/**/*.rb").sort.each do |file|
82
+ unless loaded_paths.include?(file)
83
+ require_dependency file
84
+ loaded_paths << file
85
+ end
86
+ end
87
+ end
88
+ @world.reload! if @world
89
+ end
90
+
91
+ def loaded_paths
92
+ @loaded_paths ||= Set.new
93
+ end
94
+ end
95
+ end
@@ -0,0 +1,146 @@
1
+ module Dynflow
2
+ class Rails
3
+ class Configuration
4
+ # the number of threads in the pool handling the execution
5
+ attr_accessor :pool_size
6
+
7
+ # the size of db connection pool
8
+ attr_accessor :db_pool_size
9
+
10
+ # set true if the executor runs externally (by default true in procution, othewise false)
11
+ attr_accessor :remote
12
+ alias remote? remote
13
+
14
+ # what transaction adapater should be used, by default, it uses the ActiveRecord
15
+ # based adapter, expecting ActiveRecord is used as ORM in the application
16
+ attr_accessor :transaction_adapter
17
+
18
+ attr_accessor :eager_load_paths
19
+
20
+ attr_accessor :lazy_initialization
21
+
22
+ # what rake tasks should run their own executor, not depending on the external one
23
+ attr_accessor :rake_tasks_with_executor
24
+
25
+ def initialize
26
+ self.pool_size = 5
27
+ self.db_pool_size = pool_size + 5
28
+ self.remote = ::Rails.env.production?
29
+ self.transaction_adapter = ::Dynflow::TransactionAdapters::ActiveRecord.new
30
+ self.eager_load_paths = []
31
+ self.lazy_initialization = !::Rails.env.production?
32
+ self.rake_tasks_with_executor = %w(db:migrate db:seed)
33
+
34
+ @on_init = []
35
+ end
36
+
37
+ # Action related info such as exceptions raised inside the actions' methods
38
+ # To be overridden in the Rails application
39
+ def action_logger
40
+ ::Rails.logger
41
+ end
42
+
43
+ # Dynflow related info about the progress of the execution
44
+ # To be overridden in the Rails application
45
+ def dynflow_logger
46
+ ::Rails.logger
47
+ end
48
+
49
+ def on_init(&block)
50
+ @on_init << block
51
+ end
52
+
53
+ def run_on_init_hooks(world)
54
+ @on_init.each { |init| init.call(world) }
55
+ end
56
+
57
+ def initialize_world(world_class = ::Dynflow::World)
58
+ world_class.new(world_config)
59
+ end
60
+
61
+ # No matter what config.remote says, when the process is marked as executor,
62
+ # it can't be remote
63
+ def remote?
64
+ !::Rails.application.dynflow.executor? &&
65
+ !rake_task_with_executor? &&
66
+ @remote
67
+ end
68
+
69
+ def rake_task_with_executor?
70
+ return false unless defined?(::Rake)
71
+
72
+ ::Rake.application.top_level_tasks.any? do |rake_task|
73
+ rake_tasks_with_executor.include?(rake_task)
74
+ end
75
+ end
76
+
77
+ def increase_db_pool_size?
78
+ !::Rails.env.test?
79
+ end
80
+
81
+ # To avoid pottential timeouts on db connection pool, make sure
82
+ # we have the pool bigger than the thread pool
83
+ def increase_db_pool_size
84
+ if increase_db_pool_size?
85
+ ::ActiveRecord::Base.connection_pool.disconnect!
86
+
87
+ config = ::ActiveRecord::Base.configurations[::Rails.env]
88
+ config['pool'] = db_pool_size if config['pool'].to_i < db_pool_size
89
+ ::ActiveRecord::Base.establish_connection(config)
90
+ end
91
+ end
92
+
93
+ # generates the options hash consumable by the Dynflow's world
94
+ def world_config
95
+ ::Dynflow::Config.new.tap do |config|
96
+ config.auto_rescue = true
97
+ config.logger_adapter = ::Dynflow::LoggerAdapters::Delegator.new(action_logger, dynflow_logger)
98
+ config.pool_size = 5
99
+ config.persistence_adapter = initialize_persistence
100
+ config.transaction_adapter = transaction_adapter
101
+ config.executor = ->(world, _) { initialize_executor(world) }
102
+ config.connector = ->(world, _) { initialize_connector(world) }
103
+
104
+ # we can't do any operation until the Rails.application.dynflow.world is set
105
+ config.auto_execute = false
106
+ end
107
+ end
108
+
109
+ protected
110
+
111
+ def default_sequel_adapter_options
112
+ db_config = ::ActiveRecord::Base.configurations[::Rails.env].dup
113
+ db_config['adapter'] = 'postgres' if db_config['adapter'] == 'postgresql'
114
+ db_config['max_connections'] = db_pool_size if increase_db_pool_size?
115
+
116
+ if db_config['adapter'] == 'sqlite3'
117
+ db_config['adapter'] = 'sqlite'
118
+ database = db_config['database']
119
+ unless database == ':memory:'
120
+ # We need to create separate database for sqlite
121
+ # to avoid lock conflicts on the database
122
+ db_config['database'] = "#{File.dirname(database)}/dynflow-#{File.basename(database)}"
123
+ end
124
+ end
125
+ db_config
126
+ end
127
+
128
+ def initialize_executor(world)
129
+ if remote?
130
+ false
131
+ else
132
+ ::Dynflow::Executors::Parallel.new(world, pool_size)
133
+ end
134
+ end
135
+
136
+ def initialize_connector(world)
137
+ ::Dynflow::Connectors::Database.new(world)
138
+ end
139
+
140
+ # Sequel adapter based on Rails app database.yml configuration
141
+ def initialize_persistence
142
+ ::Dynflow::PersistenceAdapters::Sequel.new(default_sequel_adapter_options)
143
+ end
144
+ end
145
+ end
146
+ end
@@ -0,0 +1,72 @@
1
+ require 'fileutils'
2
+
3
+ module Dynflow
4
+ class Rails
5
+ class Daemon
6
+ # Load the Rails environment and initialize the executor in this thread.
7
+ def run(rails_root = Dir.pwd)
8
+ STDOUT.puts('Starting Rails environment')
9
+ rails_env_file = File.expand_path('./config/environment.rb', rails_root)
10
+ unless File.exist?(rails_env_file)
11
+ raise "#{rails_root} doesn't seem to be a Rails root directory"
12
+ end
13
+ ::Rails.application.dynflow.executor!
14
+ require rails_env_file
15
+ STDOUT.puts('Everything ready')
16
+ sleep
17
+ ensure
18
+ STDOUT.puts('Exiting')
19
+ end
20
+
21
+ # run the executor as a daemon
22
+ def run_background(command = 'start', options = {})
23
+ default_options = { rails_root: Dir.pwd,
24
+ process_name: 'dynflow_executor',
25
+ pid_dir: File.join(::Rails.root, 'tmp', 'pids'),
26
+ log_dir: File.join(::Rails.root, 'log'),
27
+ wait_attempts: 300,
28
+ wait_sleep: 1,
29
+ executors_count: (ENV['EXECUTORS_COUNT'] || 1).to_i }
30
+ options = default_options.merge(options)
31
+ FileUtils.mkdir_p(options[:pid_dir])
32
+ begin
33
+ require 'daemons'
34
+ rescue LoadError
35
+ raise "You need to add gem 'daemons' to your Gemfile if you wish to use it."
36
+ end
37
+
38
+ unless %w(start stop restart run).include?(command)
39
+ raise "Command exptected to be 'start', 'stop', 'restart', 'run', was #{command.inspect}"
40
+ end
41
+
42
+ STDOUT.puts("Dynflow Executor: #{command} in progress")
43
+
44
+ options[:executors_count].times do
45
+ Daemons.run_proc(options[:process_name],
46
+ :multiple => true,
47
+ :dir => options[:pid_dir],
48
+ :log_dir => options[:log_dir],
49
+ :dir_mode => :normal,
50
+ :monitor => true,
51
+ :log_output => true,
52
+ :ARGV => [command]) do |*_args|
53
+ begin
54
+ ::Logging.reopen
55
+ run(options[:rails_root])
56
+ rescue => e
57
+ STDERR.puts e.message
58
+ ::Rails.logger.exception('Failed running Dynflow daemon', e)
59
+ exit 1
60
+ end
61
+ end
62
+ end
63
+ end
64
+
65
+ protected
66
+
67
+ def world
68
+ ::Rails.application.dynflow.world
69
+ end
70
+ end
71
+ end
72
+ end
@@ -1,3 +1,3 @@
1
1
  module Dynflow
2
- VERSION = '0.8.19'
2
+ VERSION = '0.8.20'
3
3
  end
@@ -0,0 +1,48 @@
1
+ require_relative 'test_helper'
2
+ require 'active_job'
3
+ require 'dynflow/active_job/queue_adapter'
4
+
5
+ module Dynflow
6
+ class SampleJob < ::ActiveJob::Base
7
+ def perform(msg)
8
+ puts "This job says #{msg}"
9
+ end
10
+ end
11
+
12
+ describe 'running jobs' do
13
+ before(:all) do
14
+ world = WorldFactory.create_world
15
+ ::ActiveJob::QueueAdapters.include(::Dynflow::ActiveJob::QueueAdapters)
16
+ ::ActiveJob::Base.queue_adapter = :dynflow
17
+ dynflow_mock = Minitest::Mock.new
18
+ dynflow_mock.expect(:world, world)
19
+ rails_app_mock = Minitest::Mock.new
20
+ rails_app_mock .expect(:dynflow, dynflow_mock)
21
+ rails_mock = Minitest::Mock.new
22
+ rails_mock.expect(:application, rails_app_mock)
23
+ ::Rails = rails_mock
24
+ end
25
+
26
+ it 'is able to run the job right away' do
27
+ out, = capture_subprocess_io do
28
+ SampleJob.perform_now 'hello'
29
+ end
30
+ assert_match(/job says hello/, out)
31
+ end
32
+
33
+ it 'enqueues the job' do
34
+ out, = capture_subprocess_io do
35
+ SampleJob.perform_later 'hello'
36
+ end
37
+ assert_match(/Enqueued Dynflow::SampleJob/, out)
38
+ end
39
+
40
+ it 'schedules job in the future' do
41
+ out, = capture_subprocess_io do
42
+ SampleJob.set(:wait => 1.seconds).perform_later 'hello'
43
+ end
44
+ assert_match(/Enqueued Dynflow::SampleJob.*at.*UTC/, out)
45
+ end
46
+ end
47
+ end
48
+
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dynflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.19
4
+ version: 0.8.20
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ivan Necas
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-01-24 00:00:00.000000000 Z
12
+ date: 2017-02-24 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: multi_json
@@ -161,16 +161,30 @@ dependencies:
161
161
  name: activerecord
162
162
  requirement: !ruby/object:Gem::Requirement
163
163
  requirements:
164
- - - ">="
164
+ - - "<"
165
165
  - !ruby/object:Gem::Version
166
- version: '0'
166
+ version: 5.0.0
167
167
  type: :development
168
168
  prerelease: false
169
169
  version_requirements: !ruby/object:Gem::Requirement
170
170
  requirements:
171
- - - ">="
171
+ - - "<"
172
172
  - !ruby/object:Gem::Version
173
- version: '0'
173
+ version: 5.0.0
174
+ - !ruby/object:Gem::Dependency
175
+ name: activejob
176
+ requirement: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "<"
179
+ - !ruby/object:Gem::Version
180
+ version: 5.0.0
181
+ type: :development
182
+ prerelease: false
183
+ version_requirements: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "<"
186
+ - !ruby/object:Gem::Version
187
+ version: 5.0.0
174
188
  - !ruby/object:Gem::Dependency
175
189
  name: sqlite3
176
190
  requirement: !ruby/object:Gem::Requirement
@@ -367,6 +381,7 @@ files:
367
381
  - lib/dynflow/action/suspended.rb
368
382
  - lib/dynflow/action/timeouts.rb
369
383
  - lib/dynflow/action/with_sub_plans.rb
384
+ - lib/dynflow/active_job/queue_adapter.rb
370
385
  - lib/dynflow/actor.rb
371
386
  - lib/dynflow/clock.rb
372
387
  - lib/dynflow/config.rb
@@ -444,6 +459,9 @@ files:
444
459
  - lib/dynflow/persistence_adapters/sequel_migrations/007_future_execution.rb
445
460
  - lib/dynflow/persistence_adapters/sequel_migrations/008_rename_scheduled_plans_to_delayed_plans.rb
446
461
  - lib/dynflow/persistence_adapters/sequel_migrations/009_fix_mysql_data_length.rb
462
+ - lib/dynflow/rails.rb
463
+ - lib/dynflow/rails/configuration.rb
464
+ - lib/dynflow/rails/daemon.rb
447
465
  - lib/dynflow/round_robin.rb
448
466
  - lib/dynflow/semaphores.rb
449
467
  - lib/dynflow/semaphores/abstract.rb
@@ -484,6 +502,7 @@ files:
484
502
  - lib/dynflow/world.rb
485
503
  - test/abnormal_states_recovery_test.rb
486
504
  - test/action_test.rb
505
+ - test/activejob_adapter.rb
487
506
  - test/clock_test.rb
488
507
  - test/concurrency_control_test.rb
489
508
  - test/coordinator_test.rb
@@ -561,6 +580,7 @@ summary: DYNamic workFLOW engine
561
580
  test_files:
562
581
  - test/abnormal_states_recovery_test.rb
563
582
  - test/action_test.rb
583
+ - test/activejob_adapter.rb
564
584
  - test/clock_test.rb
565
585
  - test/concurrency_control_test.rb
566
586
  - test/coordinator_test.rb