burstflow 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 24e4b83edc403cfa7903a0aa9fe90ab8fe0968d5
-  data.tar.gz: ff4e300d06ab5152d9ed07913b6cb88c73afefd6
+SHA256:
+  metadata.gz: 435e3f0b3dba1c654d192d3f5f4cb1687bfe55e6c60f03e6fcb4643946603f0a
+  data.tar.gz: e5390d5f58fb4a2e3503516f0c0744bce0cac68411edf56477e82d84a9f9ed87
 SHA512:
-  metadata.gz: d123b0cde794b44e1d5c969084823a299033f0d551a8a4f2790bb1622914abdb19ae3d11b1f29f9cf9c3e97edd5a68423aae8ecb785308c0e04d34507390f468
-  data.tar.gz: e9af6f93dcae06d5d60ec1f71d746de2d9ef49e783f081a76fe854085686355e53a040978262d22ef1df7534500fca1a7caf7644cc95e531eaab9ab9aac5c19c
+  metadata.gz: fcb37c6b28837184136c25333cfaa3bd403bb953b9aa8dc8fe5971d499013650664444b5415aa50e2530218a88d07107e5032f90cc3ef3cd5114bd8593752669
+  data.tar.gz: ae739a9ab5a48b10512c3461e4720e54e564722192fca3aaf36123a48ed3a1c36de28483d19fd975659b2586637120be3dd32068f62e5bb2a294a1f6c8177e33

data/Gemfile

@@ -5,12 +5,10 @@ platforms :mri, :ruby do
   gem 'yajl-ruby'
 end
 
-gem 'rake'
-
 group :test do
   gem 'activesupport'
-  gem 'awesome_print'
   gem 'database_cleaner'
   gem 'otr-activerecord'
   gem 'pg', '~> 0.21.0'
+  gem 'awesome_print'
 end

data/Gemfile.lock

@@ -0,0 +1,119 @@
+PATH
+  remote: .
+  specs:
+    burstflow (0.2.0)
+      activejob
+      activerecord
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (5.2.2)
+      actionview (= 5.2.2)
+      activesupport (= 5.2.2)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.2.2)
+      activesupport (= 5.2.2)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.3)
+    activejob (5.2.2)
+      activesupport (= 5.2.2)
+      globalid (>= 0.3.6)
+    activemodel (5.2.2)
+      activesupport (= 5.2.2)
+    activerecord (5.2.2)
+      activemodel (= 5.2.2)
+      activesupport (= 5.2.2)
+      arel (>= 9.0)
+    activesupport (5.2.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    arel (9.0.0)
+    awesome_print (1.8.0)
+    builder (3.2.3)
+    concurrent-ruby (1.1.4)
+    crass (1.0.4)
+    database_cleaner (1.7.0)
+    diff-lcs (1.3)
+    erubi (1.8.0)
+    generator_spec (0.9.4)
+      activesupport (>= 3.0.0)
+      railties (>= 3.0.0)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    hashie (3.6.0)
+    hashie-forbidden_attributes (0.1.1)
+      hashie (>= 3.0)
+    i18n (1.4.0)
+      concurrent-ruby (~> 1.0)
+    loofah (2.2.3)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    method_source (0.9.2)
+    mini_portile2 (2.4.0)
+    minitest (5.11.3)
+    nokogiri (1.10.1)
+      mini_portile2 (~> 2.4.0)
+    otr-activerecord (1.2.6)
+      activerecord (>= 4.0, < 5.3)
+      hashie-forbidden_attributes (~> 0.1)
+    pg (0.21.0)
+    rack (2.0.6)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
+    railties (5.2.2)
+      actionpack (= 5.2.2)
+      activesupport (= 5.2.2)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.19.0, < 2.0)
+    rake (12.3.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    thor (0.20.3)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  awesome_print
+  bundler
+  burstflow!
+  database_cleaner
+  generator_spec
+  otr-activerecord
+  pg (~> 0.21.0)
+  rake
+  rspec
+  yajl-ruby
+
+BUNDLED WITH
+   1.17.2

data/burstflow.gemspec

@@ -1,14 +1,16 @@
-lib = File.expand_path('lib', __dir__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+$:.push File.expand_path("lib", __dir__)
+
+# Maintain your gem's version:
+require "burstflow/version"
 
 Gem::Specification.new do |spec|
   spec.name          = 'burstflow'
-  spec.version       = '0.1.1'
+  spec.version       = Burstflow::VERSION
   spec.authors       = ['Samoilenko Yuri']
   spec.email         = ['kinnalru@gmail.com']
-  spec.summary       = 'Burst is a parallel workflow runner using ActiveRecord and ActiveJob'
-  spec.description   = 'Burst is a parallel workflow runner using ActiveRecord and ActiveJob. It has dependency, result pipelining and suspend/resume ability'
-  spec.homepage      = 'https://github.com/RnD-Soft/burst'
+  spec.summary       = 'Burstflow is a parallel workflow runner using ActiveRecord and ActiveJob'
+  spec.description   = 'Burstflow is a parallel workflow runner using ActiveRecord and ActiveJob. It has dependency, result pipelining and suspend/resume ability'
+  spec.homepage      = 'https://github.com/RnD-Soft/burstflow'
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0")
@@ -17,7 +19,9 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activejob'
   spec.add_dependency 'activerecord'
+  
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
+  spec.add_development_dependency  'generator_spec'
 end

data/config/database.yml

@@ -3,13 +3,14 @@ development: &base
   host:     <%= ENV['DATABASE_HOST'] || 'localhost' %>
   port:     <%= ENV['DATABASE_PORT'] || 5432 %>
   encoding: unicode
-  database: <%= ENV['DATABASE_NAME'] || 'burst_test' %>
-  pool:     4
+  database: <%= ENV['DATABASE_NAME'] || 'burstflow_test' %>
+  pool:     150
   username: <%= ENV['DATABASE_USER'] || 'postgres' %>
   password: <%= ENV['DATABASE_PASS'] || '' %>
 
 test:
- <<: *base
+  <<: *base
+
 
 production:
  <<: *base

data/db/migrate/20180101000001_create_workflow.rb

@@ -0,0 +1 @@
+../../lib/generators/burstflow/install/templates/create_workflow.rb

data/db/schema.rb

@@ -10,14 +10,20 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20_180_101_000_001) do
+ActiveRecord::Schema.define(version: 2018_01_01_000001) do
+
   # These are extensions that must be enabled in order to support this database
-  enable_extension 'pgcrypto'
-  enable_extension 'plpgsql'
+  enable_extension "pgcrypto"
+  enable_extension "plpgsql"
 
-  create_table 'burst_workflows', id: :uuid, default: -> { 'gen_random_uuid()' }, force: :cascade do |t|
-    t.jsonb 'flow', default: {}, null: false
-    t.datetime 'created_at', null: false
-    t.datetime 'updated_at', null: false
+  create_table "burstflow_workflows", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
+    t.string "type"
+    t.string "status"
+    t.jsonb "flow", default: {}, null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["status"], name: "index_burstflow_workflows_on_status"
+    t.index ["type"], name: "index_burstflow_workflows_on_type"
   end
+
 end

data/lib/burstflow.rb

@@ -0,0 +1,11 @@
+require 'burstflow/railtie' if defined?(Rails)
+
+require 'active_support/all'
+
+require 'burstflow/workflow'
+require 'burstflow/job'
+require 'burstflow/worker'
+
+module Burstflow
+
+end

data/lib/burstflow/job.rb

@@ -0,0 +1,102 @@
+require "active_support/rescuable"
+require "active_job/callbacks"
+
+class Burstflow::Job
+  require "burstflow/job/exception"
+  require "burstflow/job/model"
+  require "burstflow/job/initialization"
+  require "burstflow/job/state"
+  require "burstflow/job/callbacks"
+
+  include Burstflow::Job::Model
+  include Burstflow::Job::Initialization
+  include Burstflow::Job::State
+  include Burstflow::Job::Callbacks
+  include ActiveSupport::Rescuable
+
+  attr_accessor :workflow, :payloads
+
+  define_stored_attributes :id, :workflow_id, :klass, :params, :incoming, :outgoing, :output, :failure
+  define_stored_attributes :enqueued_at, :started_at, :finished_at, :failed_at, :suspended_at, :resumed_at
+
+  SUSPEND = 'suspend'.freeze
+
+  def save!
+    workflow.with_lock do
+      workflow.set_job(self)
+      workflow.save!
+      yield if block_given?
+    end
+  end
+
+  # execute this code by ActiveJob. You may return Burstflow::Job::SUSPEND to suspend job, or call suspend method
+  def perform
+  end
+
+  def perform_now
+    run_callbacks :perform do
+      perform
+    end
+  rescue => exception
+    rescue_with_handler(exception) || raise
+  end
+
+  # execute this code when resumes after suspending
+  def resume(data)
+    raise InternalError.new(self, "Can't perform resume: not resumed") if !resumed?
+    set_output(data)
+  end
+
+  def resume_now data
+    run_callbacks :resume do
+      resume(data)
+    end
+  rescue => exception
+    rescue_with_handler(exception) || raise
+  end
+
+  # store data to be available for next jobs
+  def set_output(data)
+    self.output = data
+  end
+
+  # mark execution as suspended
+  def suspend
+    raise InternalError.new(self, "Can't suspend: not running") if !running?
+    set_output(SUSPEND)
+  end
+
+  def configure *args, &block
+    workflow.with_lock do
+
+      builder = Burstflow::Workflow::Builder.new(workflow, *args, &block)
+      workflow.flow['jobs_config'] = builder.as_json
+      workflow.save!
+      reload
+    end
+  end
+
+  def attributes
+    {
+      workflow_id: self.workflow_id,
+      id: self.id,
+      klass: self.klass,
+      params: params,
+      incoming: self.incoming,
+      outgoing: self.outgoing,
+      output: output,
+      started_at: started_at,
+      enqueued_at: enqueued_at,
+      finished_at: finished_at,
+      failed_at: failed_at,
+      suspended_at: suspended_at,
+      resumed_at: resumed_at,
+      failure: self.failure
+    }
+  end
+
+  
+
+
+
+end

data/lib/burstflow/job/callbacks.rb

@@ -0,0 +1,55 @@
+module Burstflow::Job::Callbacks
+  extend  ActiveSupport::Concern
+  include ActiveJob::Callbacks
+
+  included do
+
+    define_callbacks :suspend
+    define_callbacks :resume
+    define_callbacks :failure
+
+  end
+
+  class_methods do
+
+    def before_suspend(*filters, &blk)
+      set_callback(:suspend, :before, *filters, &blk)
+    end
+
+    def after_suspend(*filters, &blk)
+      set_callback(:suspend, :after, *filters, &blk)
+    end
+
+    def around_suspend(*filters, &blk)
+      set_callback(:suspend, :around, *filters, &blk)
+    end
+
+  
+    def before_resume(*filters, &blk)
+      set_callback(:resume, :before, *filters, &blk)
+    end
+
+    def after_resume(*filters, &blk)
+      set_callback(:resume, :after, *filters, &blk)
+    end
+
+    def around_resume(*filters, &blk)
+      set_callback(:resume, :around, *filters, &blk)
+    end
+
+  
+    def before_failure(*filters, &blk)
+      set_callback(:failure, :before, *filters, &blk)
+    end
+  
+    def after_failure(*filters, &blk)
+      set_callback(:failure, :after, *filters, &blk)
+    end
+    
+    def around_failure(*filters, &blk)
+      set_callback(:failure, :around, *filters, &blk)
+    end
+
+  end
+
+end

data/lib/burstflow/job/exception.rb

@@ -0,0 +1,8 @@
+class Burstflow::Job::InternalError < ::RuntimeError
+  attr_accessor :job
+
+  def initialize(job, message)
+    @job = job
+    super(message)
+  end
+end

data/lib/burstflow/job/initialization.rb

@@ -0,0 +1,35 @@
+module Burstflow::Job::Initialization
+  extend  ActiveSupport::Concern
+
+  included do
+
+    def initialize(workflow, job_config = {})
+      @workflow = workflow
+      assign_default_values(job_config)
+    end
+
+    def assign_default_values(job_config)
+      set_model(job_config.deep_dup)
+
+      self.id ||= SecureRandom.uuid
+      self.workflow_id ||= @workflow.try(:id)
+      self.klass ||= self.class.to_s
+      self.incoming ||= []
+      self.outgoing ||= []
+    end
+
+    def reload
+      assign_default_values(@workflow.job_hash(self.id))
+    end
+
+  end
+
+  class_methods do
+
+    def from_hash(workflow, job_config)
+      job_config[:klass].constantize.new(workflow, job_config)
+    end
+
+  end
+
+end