dynflow 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef3a326c234f3687c367017a5a64befabfb11075944eed324b5babc8b7adec95
-  data.tar.gz: 615bad06c6a5223613b974e1d7d3ae5c6c9e87d53aaf4d038678ec64168cc498
+  metadata.gz: bb02c56f3fda535ffdc29d64826470d8bb81cf4714ac98e8316dbcb0f045ce00
+  data.tar.gz: 7f0553a6e9bda6a5ae8d604f5f6238b52e5b6ad45462a3199c993a30e530f499
 SHA512:
-  metadata.gz: 1307e77ffb5874d7d7ee4bfc010d85c69de267420e6273790971a54921144114afd9d6c02885213feb8b67fca0de28a8835b149f628330ed7976139842764568
-  data.tar.gz: ceda3668f29ff0aac123d9c767c327dda98822dfdf4d8aca9d6e03df7f28bb7c41a60300f4ee25c4348e2789c1d34cf651d692d196dfe2a0fde4be6c888825be
+  metadata.gz: 9d3bdd0745db1d85f1be3c1c4510a54d167d175b48d33e8a89fbeaf7fa7c115f419a8756b89af4f2f4d57bad94a2bb1c999f8d6d9d530b850b7dab5a1638b042
+  data.tar.gz: 022b09ce7daa8ed8a2a856d5cc16f93c9375978d9fc571350dbaebe2c7c6381dadcc8c3510e6227c08872e85657ad8e27ebfbb3c70f5e0f1df61b946dfa3f4c1

data/.rubocop.yml

@@ -1,7 +1,7 @@
 inherit_from: .rubocop_todo.yml
 
 AllCops:
-  TargetRubyVersion: 2.2
+  TargetRubyVersion: 2.3
 
 # Cop supports --auto-correct.
 # Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.

data/.travis.yml

@@ -1,9 +1,6 @@
-sudo: false
-language:
-  - ruby
+language: ruby
 
 services:
-  - mysql
   - postgresql
 
 rvm:
@@ -19,6 +16,8 @@ matrix:
   include:
     - rvm: "2.4.0"
       env: "DB=mysql DB_CONN_STRING=mysql2://root@localhost/travis_ci_test"
+      services:
+        - mysql
     - rvm: "2.4.0"
       env: "DB=sqlite3 DB_CONN_STRING=sqlite:/"
     - rvm: "2.4.0"

data/Dockerfile

@@ -0,0 +1,9 @@
+FROM fedora:30
+RUN dnf install -y curl git gcc gcc-c++ make openssl bzip2 findutils openssl-devel readline-devel \
+                   zlib-devel sqlite-devel ruby ruby-devel rubygem-bundler redhat-rpm-config libpqxx-devel
+
+ADD Gemfile /data/
+ADD dynflow.gemspec /data/
+ADD lib/dynflow/version.rb /data/lib/dynflow/version.rb
+WORKDIR /data
+RUN bundle install --without mysql

data/Gemfile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 source 'https://rubygems.org'
 
 gemspec
@@ -11,6 +12,11 @@ group :pry do
   gem 'pry-byebug'
 end
 
+group :sidekiq do
+  gem 'sidekiq'
+  gem 'gitlab-sidekiq-fetcher', :require => 'sidekiq-reliable-fetch'
+end
+
 group :postgresql do
   gem "pg"
 end

data/Rakefile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'rake/testtask'
 require 'fileutils'
 

data/doc/pages/Gemfile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 source 'https://rubygems.org'
 
 gem 'rake'

data/doc/pages/Rakefile

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'rubygems'
 require 'rake'
 

data/doc/pages/plugins/alert_block.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require_relative 'div_tag'
 
 module Jekyll

data/doc/pages/plugins/div_tag.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Jekyll
   class DivTag < Liquid::Block
     def render(context)

data/doc/pages/plugins/graphviz.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # taken from https://raw.githubusercontent.com/kui/octopress-graphviz/master/graphviz_block.rb
 
 require 'open3'

data/doc/pages/plugins/plantuml.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Title: PlantUML Code Blocks for Jekyll
 # Author: YJ Park (yjpark@gmail.com)
 # https://github.com/yjpark/jekyll-plantuml

data/doc/pages/plugins/play.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'pp'
 require 'pry'
 

data/doc/pages/plugins/tags.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'nuggets/range/quantile'
 require 'erb'
 

data/doc/pages/plugins/toc.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Jekyll
   module FancyToCFilter
     def fancytoc(input)

data/docker-compose.yml

@@ -0,0 +1,41 @@
+version: '3'
+services:
+  postgres:
+    image: postgres:11-alpine
+    expose:
+      - 5432
+    environment:
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=dynflow
+  redis:
+    image: redis:5.0.5-alpine
+    expose:
+      - 6379
+  client: &common
+    build: .
+    depends_on:
+      - postgres
+      - redis
+    environment:
+      - REDIS_URL=redis://redis:6379/0
+      - REDIS_PROVIDER=REDIS_URL
+      - DB_CONN_STRING=postgresql://postgres:postgres@postgres:5432/dynflow
+    volumes:
+      - ./:/data
+    command: |
+      ruby ./examples/remote_executor.rb client
+  observer:
+    <<: *common
+    command: |
+      ruby ./examples/remote_executor.rb observer
+    ports:
+      - 4567:4567
+  orchestrator:
+    <<: *common
+    command: |
+      sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator -c 1
+  worker:
+    <<: *common
+    command: |
+      sidekiq -r ./examples/remote_executor.rb -q default

data/dynflow.gemspec

@@ -1,4 +1,5 @@
 # -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 $:.push File.expand_path("../lib", __FILE__)
 require "dynflow/version"
 

data/examples/clock_benchmark.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'dynflow'
 require 'benchmark'
 

data/examples/example_helper.rb

@@ -1,8 +1,13 @@
+# frozen_string_literal: true
 $:.unshift(File.expand_path('../../lib', __FILE__))
 
 require 'dynflow'
 
 class ExampleHelper
+  CONSOLE_URL='http://localhost:4567'
+  DYNFLOW_URL="#{CONSOLE_URL}/dynflow"
+  SIDEKIQ_URL="#{CONSOLE_URL}/sidekiq"
+
   class << self
     def world
       @world ||= create_world
@@ -18,6 +23,7 @@ class ExampleHelper
       config.logger_adapter      = logger_adapter
       config.auto_rescue         = false
       config.telemetry_adapter   = telemetry_adapter
+      config.queues.add(:slow, :pool_size => 3)
       yield config if block_given?
       Dynflow::World.new(config).tap do |world|
         puts "World #{world.id} started..."
@@ -41,7 +47,7 @@ class ExampleHelper
     end
 
     def logger_adapter
-      Dynflow::LoggerAdapters::Simple.new $stderr, 4
+      Dynflow::LoggerAdapters::Simple.new $stderr, Logger::FATAL
     end
 
     def run_web_console(world = ExampleHelper.world)
@@ -49,7 +55,18 @@ class ExampleHelper
       dynflow_console = Dynflow::Web.setup do
         set :world, world
       end
-      Rack::Server.new(:app => dynflow_console, :Port => 4567).start
+      apps = { '/dynflow' => dynflow_console }
+      puts "Starting Dynflow console at #{DYNFLOW_URL}"
+      begin
+        require 'sidekiq/web'
+        apps['/sidekiq'] = Sidekiq::Web
+        puts "Starting Sidekiq console at #{SIDEKIQ_URL}"
+      rescue LoadError
+        puts 'Sidekiq not around, not mounting the console'
+      end
+
+      app = Rack::URLMap.new(apps)
+      Rack::Server.new(:app => app, :Host => '0.0.0.0', :Port => URI.parse(CONSOLE_URL).port).start
     end
 
     # for simulation of the execution failing for the first time

data/examples/future_execution.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require_relative 'example_helper'
 
@@ -64,7 +65,7 @@ if $0 == __FILE__
     |    #{near_future_plan.id} is delayed to execute at #{near_future} and should run successfully.
     |    #{future_plan.id} is delayed to execute at #{future} and should run successfully.
     |
-    |  Visit http://localhost:4567 to see their status.
+    |  Visit #{ExampleHelper::DYNFLOW_URL} to see their status.
     |
   MSG
 

data/examples/memory_limit_watcher.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require_relative 'example_helper'
 

data/examples/orchestrate.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 example_description = <<DESC
   Orchestrate Example
@@ -15,7 +16,7 @@ example_description = <<DESC
   It also simulates a failure and demonstrates the Dynflow ability to rescue
   from the error and consinue with the run.
 
-  Once the Sinatra web console starts, you can navigate to http://localhost:4567
+  Once the Sinatra web console starts, you can navigate to #{ExampleHelper::DYNFLOW_URL}
   to see what's happening in the Dynflow world.
 
 DESC
@@ -135,7 +136,7 @@ module Orchestrate
         puts <<-MSG.gsub(/^.*\|/, '')
 
                 | Execution plan #{execution_plan_id} is failing
-                | You can resume it at http://localhost:4567/#{execution_plan_id}
+                | You can resume it at #{ExampleHelper::DYNFLOW_URL}/#{execution_plan_id}
 
         MSG
         raise "temporary unavailabe"
@@ -148,9 +149,7 @@ module Orchestrate
 end
 
 if $0 == __FILE__
-  world = ExampleHelper.create_world do |config|
-    config.queues.add(:slow, :pool_size => 3)
-  end
+  world = ExampleHelper.create_world
   ExampleHelper.set_world(world)
   ExampleHelper.world.action_logger.level = Logger::INFO
   ExampleHelper.something_should_fail!

data/examples/orchestrate_evented.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require_relative 'example_helper'
 
@@ -17,7 +18,7 @@ example_description = <<DESC
   send external events to the action while it's suspended. One use case is being
   able to cancel the action while it's running.
 
-  Once the Sinatra web console starts, you can navigate to http://localhost:4567
+  Once the Sinatra web console starts, you can navigate to #{ExampleHelper::DYNFLOW_URL}
   to see what's happening in the Dynflow world.
 
 DESC
@@ -154,7 +155,7 @@ module OrchestrateEvented
 
             | Execution plan #{execution_plan_id} got stuck
             | You can cancel the stucked step at
-            | http://localhost:4567/#{execution_plan_id}
+            | #{ExampleHelper::DYNFLOW_URL}/#{execution_plan_id}
 
         MSG
         # we suspend the action but don't plan the wakeup event,

data/examples/remote_executor.rb

@@ -1,5 +1,31 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
+# frozen_string_literal: true
+
+# To run the observer:
+#
+#      bundle exec ruby ./examples/remote_executor.rb observer
+#
+# To run the server:
+#
+#      bundle exec ruby ./examples/remote_executor.rb server
+#
+# To run the client:
+#
+#      bundle exec ruby ./examples/remote_executor.rb client
+#
+# Sidekiq
+# =======
+#
+# In case of using Sidekiq as async job backend, use this instead of the server command
+#
+# To run the orchestrator:
+#
+#      bundle exec sidekiq -r ./examples/remote_executor.rb -q dynflow_orchestrator
+#
+# To run the worker:
+#
+#      bundle exec sidekiq -r ./examples/remote_executor.rb -q default
 
 require_relative 'example_helper'
 require_relative 'orchestrate_evented'
@@ -37,6 +63,34 @@ class RemoteExecutorExample
       run(world)
     end
 
+    def initialize_sidekiq_orchestrator
+      ExampleHelper.create_world do |config|
+        config.persistence_adapter = persistence_adapter
+        config.connector           = connector
+        config.executor            = ::Dynflow::Executors::Sidekiq::Core
+        config.auto_validity_check = false
+      end
+    end
+
+    def initialize_sidekiq_worker
+      Sidekiq.configure_server do |config|
+        require 'sidekiq-reliable-fetch'
+        # Use semi-reliable fetch
+        # for details see https://gitlab.com/gitlab-org/sidekiq-reliable-fetch/blob/master/README.md
+        config.options[:semi_reliable_fetch] = true
+        # Do not requeue jobs after sidekiq shutdown
+        config.options[:max_retries_after_interruption] = 0
+        # Do not store interrupted jobs, just discard them
+        config.options[:interrupted_max_jobs] = 0
+        Sidekiq::ReliableFetch.setup_reliable_fetch!(config)
+      end
+      ExampleHelper.create_world do |config|
+        config.persistence_adapter = persistence_adapter
+        config.connector           = connector
+        config.executor            = false
+      end
+    end
+
     def run(world)
       begin
         ExampleHelper.run_web_console(world)
@@ -75,7 +129,10 @@ class RemoteExecutorExample
       world.trigger(OrchestrateEvented::CreateInfrastructure, true)
 
       loop do
+        start_time = Time.now
         world.trigger(SampleAction).finished.wait
+        finished_in = Time.now - start_time
+        puts "Finished in #{finished_in}s"
         sleep 0.5
       end
     end
@@ -106,4 +163,15 @@ MSG
     puts "Unknown command #{comment}"
     exit 1
   end
+elsif defined?(Sidekiq)
+  # TODO:
+  Sidekiq.default_worker_options = { :retry => 0, 'backtrace' => true }
+  # assuming the remote executor was required as part of initialization
+  # of the ActiveJob worker
+  world = if Sidekiq.options[:queues].include?("dynflow_orchestrator")
+            RemoteExecutorExample.initialize_sidekiq_orchestrator
+          elsif (Sidekiq.options[:queues] - ['dynflow_orchestrator']).any?
+            RemoteExecutorExample.initialize_sidekiq_worker
+          end
+  Sidekiq.options[:dynflow_world] = world
 end

data/examples/singletons.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 example_description = <<DESC
 
   Sub Plans Example
@@ -44,9 +45,9 @@ if $0 == __FILE__
     |  #{t2.id} should fail because #{t1.id} holds the lock
     |
     |  You can see the details at
-    |    http://localhost:4567/#{t1.id}
-    |    http://localhost:4567/#{t2.id}
-    |    http://localhost:4567/#{t3.id}
+    |    #{ExampleHelper::DYNFLOW_URL}/#{t1.id}
+    |    #{ExampleHelper::DYNFLOW_URL}/#{t2.id}
+    |    #{ExampleHelper::DYNFLOW_URL}/#{t3.id}
     |
   MSG
   ExampleHelper.run_web_console

data/examples/sub_plan_concurrency_control.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 example_description = <<DESC
   Sub-plan Concurrency Control Example
@@ -62,7 +63,7 @@ if $0 == __FILE__
   puts example_description
   puts <<-MSG.gsub(/^.*\|/, '')
     |  Execution plan #{triggered.id} with 10 sub plans triggered
-    |  You can see the details at http://localhost:4567/#{triggered.id}/actions/1/sub_plans
+    |  You can see the details at #{ExampleHelper::DYNFLOW_URL}/#{triggered.id}/actions/1/sub_plans
     |  Or simply watch in the console that there are never more than #{ConcurrencyControlExample::ConcurrencyLevel} running at the same time.
     |
     |  The tasks are distributed over #{ConcurrencyControlExample::RunWithin} seconds.

data/examples/sub_plans.rb

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 example_description = <<DESC
   Sub Plans Example
   ===================
@@ -51,8 +52,8 @@ if $0 == __FILE__
   puts <<-MSG.gsub(/^.*\|/, '')
     |  Execution plans #{t1.id} and #{t2.id} with sub plans triggered
     |  You can see the details at
-    |    http://localhost:4567/#{t2.id}
-    |    http://localhost:4567/#{t1.id}
+    |    #{ExampleHelper::DYNFLOW_URL}/#{t2.id}
+    |    #{ExampleHelper::DYNFLOW_URL}/#{t1.id}
   MSG
 
   ExampleHelper.run_web_console