dwf 0.1.1 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a76f04523159e059efdc750a1b1a3852911836043fda70e18c50e9c0ab3ba71
-  data.tar.gz: 5a1b834705c863daf76252822fe8728646b1ec87a8dc5127fc337ef6c46001e1
+  metadata.gz: 7cfd1989fd7228daec1f81759e249cb3a0d6e920e465b38f1f6998239c78f6c7
+  data.tar.gz: 0e4139ec4372ef7f5fb91b61cae9b2131e7e5b5983e25219a213b3edb821c9c6
 SHA512:
-  metadata.gz: c7d3df090f78652a84e5929678f3f7d355966744f33b25b6a5b8f754b20e0a366bfd8be0f911d6aa2afd923b3c46f057e9d97038971c0b14efde60c16647a93a
-  data.tar.gz: 8bda6afd8f6e887c2542091c81cb5d4a5543e5a6d12b8dc74b6ac1a58e69fab75a1af1b34a262d3a922ef941c1faba5325e2191851a33509891ad953168d370e
+  metadata.gz: 38b96a776878ddce8df1757ecdba3452a833f117536b9a3f44a07dc801ce71be7ac7cfac997622cd3e172c1a2929ac333f50e4823f901cb844149189c18563de
+  data.tar.gz: 3a32f7469c5a7009e8f04a1d1c9e35fcd8de4fb322593f8e3c9f90fa617fe96eda8dc768dd97c5a3f7bb9044a2eaa87632970fcdddfd6bb57e96c6d72446951b

data/.github/workflows/build_gem.yaml

@@ -0,0 +1,37 @@
+name: Ruby Gem
+
+on:
+  # Manually publish
+  workflow_dispatch:
+  # Alternatively, publish whenever changes are merged to the `main` branch.
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 3.0.0
+        uses: ruby/setup-ruby@477b21f02be01bcb8030d50f37cfec92bfa615b6
+        with:
+          ruby-version: 3.0.0
+      - run: bundle install
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.github/workflows/test.yaml

@@ -0,0 +1,25 @@
+name: dwf
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@477b21f02be01bcb8030d50f37cfec92bfa615b6
+        with:
+          ruby-version: 3.0.0
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rspec --require spec_helper

data/.gitignore

@@ -1,6 +1,6 @@
 .byebug_history
 tmp/
 Gemfile.lock
-wf-*.gem
+dwf-*.gem
 :w
 :W

data/.rspec

@@ -0,0 +1,3 @@
+--require spec_helper
+--color
+--format documentation

data/Gemfile

@@ -2,7 +2,7 @@ source "https://rubygems.org"
 
 gemspec
 
-source "https://gems.contribsys.com/" do
-  gem 'sidekiq-pro'
-end
+# source "https://gems.contribsys.com/" do
+  # gem 'sidekiq-pro'
+# end
 

data/README.md

@@ -1,40 +1,50 @@
 # DSL playground
 [Gush](https://github.com/chaps-io/gush) cloned without [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) but requried [Sidekiq](https://github.com/mperham/sidekiq). This project is for researching DSL purpose
 
-# Execute flow
-## Declare jobs
+# Installation
+## 1. Add `dwf` to Gemfile
+```ruby
+gem 'dwf', '~> 0.1.3'
+```
+## 2. Execute flow
+### Declare jobs
 
 ```ruby
-require_relative './wf/item'
+require 'dwf'
 
-class A < Wf::Item
+class A < Dwf::Item
   def perform
     puts "#{self.class.name} Working"
     sleep 2
+    puts params
     puts "#{self.class.name} Finished"
   end
 end
-
-class E < A; end
-class B < A; end
-class C < E; end
-class D < E; end
 ```
 
-## Declare flow
+### Declare flow
 ```ruby
-class TestWf < Wf::Workflow
+require 'dwf'
+
+class TestWf < Dwf::Workflow
   def configure
     run A
     run B, after: A
     run C, after: A
-    run E, after: [B, C]
-    run D, after: [E]
+    run E, after: [B, C], params: 'E say hello'
+    run D, after: [E], params: 'D say hello'
+    run F, params: 'F say hello'
   end
 end
-
 ```
 
+#### Note
+`dwf` supports 2 type of callback `Dwf::Workflow::BUILD_IN` and `Dwf::Workflow::SK_BATCH`
+- `Dwf::Workflow::BUILD_IN` is a build-in callback
+- `Dwf::Workflow::SK_BATCH` is [sidekiq batch](https://github.com/mperham/sidekiq/wiki/Batches) callback which required [`sidekiq-pro`](https://sidekiq.org/products/pro.html)
+
+By default `dwf` will use `Dwf::Workflow::BUILD_IN` callback.
+
 ### Execute flow
 ```ruby
 wf = TestWf.create
@@ -44,17 +54,30 @@ wf.start!
 ### Output
 ```
 A Working
+F Working
 A Finished
-B Working
+F say hello
+F Finished
 C Working
-B Finished
+B Working
 C Finished
+B Finished
 E Working
+E say hello
 E Finished
 D Working
+D say hello
 D Finished
 ```
 
+# Todo
+- [x] Make it work
+- [x] Support pass params
+- [x] Support with build-in callback
+- [x] Add github workflow
+- [ ] [WIF] Test
+- [ ] Transfer output through each node
+
 # References
 - https://github.com/chaps-io/gush
 - https://github.com/mperham/sidekiq

data/dwf.gemspec

@@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "dwf"
-  spec.version       = '0.1.1'
+  spec.version       = '0.1.5'
   spec.authors       = ["dthtien"]
   spec.email         = ["tiendt2311@gmail.com"]
 

data/lib/dwf/callback.rb

@@ -1,4 +1,3 @@
-require 'sidekiq-pro'
 require_relative 'client'
 
 module Dwf

data/lib/dwf/item.rb

@@ -1,22 +1,27 @@
+# frozen_string_literal: true
 require_relative 'client'
 
 module Dwf
   class Item
+    DEFAULT_QUEUE = 'default'
+
     attr_reader :workflow_id, :id, :params, :queue, :klass, :started_at,
-      :enqueued_at, :finished_at, :failed_at
-    attr_accessor :incomming, :outgoing
+      :enqueued_at, :finished_at, :failed_at, :callback_type
+    attr_accessor :incoming, :outgoing
 
     def initialize(options = {})
       @workflow_id = options[:workflow_id]
       @id = options[:id]
       @params = options[:params]
-      @queue = options[:queue] || 'default'
-      @incomming = options[:incoming] || []
+      @queue = options[:queue] || DEFAULT_QUEUE
+      @incoming = options[:incoming] || []
       @outgoing = options[:outgoing] || []
       @klass = options[:klass] || self.class
+      @failed_at = options[:failed_at]
       @finished_at = options[:finished_at]
       @enqueued_at = options[:enqueued_at]
       @started_at = options[:started_at]
+      @callback_type = options[:callback_type]
     end
 
     def self.from_hash(hash)
@@ -31,6 +36,10 @@ module Dwf
 
     def perform; end
 
+    def cb_build_in?
+      callback_type == Dwf::Workflow::BUILD_IN
+    end
+
     def perform_async
       Dwf::Worker.set(queue: queue).perform_async(workflow_id, name)
     end
@@ -40,11 +49,11 @@ module Dwf
     end
 
     def no_dependencies?
-      incomming.empty?
+      incoming.empty?
     end
 
     def parents_succeeded?
-      incomming.all? do |name|
+      incoming.all? do |name|
         client.find_job(workflow_id, name).succeeded?
       end
     end
@@ -125,14 +134,15 @@ module Dwf
         id: id,
         klass: klass.to_s,
         queue: queue,
-        incoming: incomming,
+        incoming: incoming,
         outgoing: outgoing,
         finished_at: finished_at,
         enqueued_at: enqueued_at,
         started_at: started_at,
         failed_at: failed_at,
         params: params,
-        workflow_id: workflow_id
+        workflow_id: workflow_id,
+        callback_type: callback_type
       }
     end
 

data/lib/dwf/worker.rb

@@ -12,7 +12,7 @@ module Dwf
       job.mark_as_started
       job.perform
       job.mark_as_finished
-      # job.enqueue_outgoing_jobs
+      job.enqueue_outgoing_jobs if job.cb_build_in?
     end
 
     private

data/lib/dwf/workflow.rb

@@ -1,32 +1,40 @@
+# frozen_string_literal: true
+
 require_relative 'client'
 require_relative 'worker'
 require_relative 'callback'
 
 module Dwf
   class Workflow
-    attr_reader :dependencies, :jobs, :started_at, :finished_at, :persisted, :stopped
+    CALLBACK_TYPES = [
+      BUILD_IN = 'build-in',
+      SK_BATCH = 'sk-batch'
+    ].freeze
+    attr_reader :dependencies, :jobs, :started_at, :finished_at, :persisted, :stopped,
+                :callback_type
 
     class << self
-      def create
-        flow = new
+      def create(*args)
+        flow = new(*args)
         flow.save
         flow
       end
     end
 
-    def initialize
+    def initialize(options = {})
       @dependencies = []
       @id = id
       @jobs = []
       @persisted = false
       @stopped = false
+      @callback_type = options[:callback_type] || BUILD_IN
 
       setup
     end
 
     def start!
       initial_jobs.each do |job|
-        Dwf::Callback.new.start(job)
+        cb_build_in? ? job.persist_and_perform_async! : Dwf::Callback.new.start(job)
       end
     end
 
@@ -37,6 +45,10 @@ module Dwf
       true
     end
 
+    def cb_build_in?
+      callback_type == BUILD_IN
+    end
+
     def id
       @id ||= client.build_workflow_id
     end
@@ -49,6 +61,7 @@ module Dwf
         id: client.build_job_id(id, klass.to_s),
         params: options.fetch(:params, {}),
         queue: options[:queue],
+        callback_type: callback_type
       )
 
       jobs << node
@@ -81,7 +94,8 @@ module Dwf
         status: status,
         stopped: stopped,
         started_at: started_at,
-        finished_at: finished_at
+        finished_at: finished_at,
+        callback_type: callback_type
       }
     end
 
@@ -143,7 +157,7 @@ module Dwf
         from = find_job(dependency[:from])
         to   = find_job(dependency[:to])
 
-        to.incomming << dependency[:from]
+        to.incoming << dependency[:from]
         from.outgoing << dependency[:to]
       end
     end

data/lib/dwf.rb

@@ -2,9 +2,9 @@
 require "bundler/setup"
 
 require 'sidekiq'
-require 'sidekiq-pro'
 require 'json'
 require 'redis'
+# require 'sidekiq-pro'
 
 require_relative 'dwf/utils'
 require_relative 'dwf/workflow'
@@ -14,6 +14,6 @@ require_relative 'dwf/worker'
 require_relative 'dwf/callback'
 
 module Dwf
-  VERSION = '0.1.0'
+  VERSION = '0.1.3'
 end
 

data/spec/dwf/item_spec.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Dwf::Item, item: true do
+  let!(:id) { SecureRandom.uuid }
+  let!(:workflow_id) { SecureRandom.uuid }
+  let(:incoming) { [] }
+  let(:outgoing) { [] }
+  let(:started_at) { nil }
+  let(:finished_at) { nil }
+  let(:options) do
+    {
+      workflow_id: workflow_id,
+      id: id,
+      params: {},
+      incoming: incoming,
+      outgoing: outgoing,
+      queue: Dwf::Item::DEFAULT_QUEUE,
+      klass: 'Dwf::Item',
+      started_at: started_at,
+      finished_at: finished_at,
+      callback_type: Dwf::Workflow::BUILD_IN
+    }
+  end
+  let!(:item) { described_class.new(options) }
+
+  describe 'self.from_hash' do
+    let!(:item) { described_class.from_hash(options) }
+    it { expect(item.to_hash.compact).to eq options.compact }
+  end
+
+  describe '#persist_and_perform_async!' do
+    let(:worker_double) { double(perform_async: nil) }
+    let(:client_double) { double(persist_job: nil) }
+
+    before do
+      allow(Dwf::Client).to receive(:new).and_return client_double
+      allow(Dwf::Worker).to receive(:set).and_return worker_double
+      item.persist_and_perform_async!
+    end
+
+    it do
+      expect(worker_double)
+        .to have_received(:perform_async)
+        .with(item.workflow_id, item.name)
+      expect(client_double).to have_received(:persist_job).with(item)
+      expect(item.enqueued_at).not_to be_nil
+    end
+  end
+
+  describe '#cb_build_in?' do
+    it { expect(item.cb_build_in?).to be_truthy }
+  end
+
+  describe '#perform_async' do
+    let(:worker_double) { double(perform_async: nil) }
+    before do
+      allow(Dwf::Worker).to receive(:set).and_return worker_double
+      item.perform_async
+    end
+
+    it do
+      expect(worker_double)
+        .to have_received(:perform_async)
+        .with(item.workflow_id, item.name)
+    end
+  end
+
+  describe '#name' do
+    it { expect(item.name).to eq "#{described_class}|#{id}" }
+  end
+
+  describe '#no_dependencies?' do
+    it { expect(item.no_dependencies?).to be_truthy }
+  end
+
+  describe '#parents_succeeded?' do
+    let(:incoming) { ["A|#{SecureRandom.uuid}"] }
+    let(:client_double) { double(find_job: nil) }
+    let(:a_item) do
+      described_class.new(
+        workflow_id: SecureRandom.uuid,
+        id: SecureRandom.uuid,
+        finished_at: finished_at
+      )
+    end
+
+    before do
+      allow(Dwf::Client).to receive(:new).and_return client_double
+      allow(client_double)
+        .to receive(:find_job).and_return a_item
+    end
+
+    context 'parent jobs already finished' do
+      let(:finished_at) { Time.now.to_i }
+
+      it do
+        expect(item.parents_succeeded?).to be_truthy
+        expect(client_double)
+          .to have_received(:find_job)
+          .with(workflow_id, incoming.first)
+      end
+    end
+
+    context 'parent jobs havent finished yet' do
+      let(:finished_at) { nil }
+
+      it do
+        expect(item.parents_succeeded?).to be_falsy
+        expect(client_double)
+          .to have_received(:find_job)
+          .with(workflow_id, incoming.first)
+      end
+    end
+  end
+
+  describe '#enqueue!' do
+    before { item.enqueue! }
+
+    it { expect(item.enqueued_at).not_to be_nil }
+  end
+
+  describe '#mark_as_started' do
+    let(:client_double) { double(persist_job: nil) }
+    before do
+      allow(Dwf::Client).to receive(:new).and_return client_double
+      item.mark_as_started
+    end
+
+    it do
+      expect(client_double).to have_received(:persist_job).with item
+      expect(item.started_at).not_to be_nil
+    end
+  end
+
+  describe '#mark_as_finished' do
+    let(:client_double) { double(persist_job: nil) }
+    before do
+      allow(Dwf::Client).to receive(:new).and_return client_double
+      item.mark_as_finished
+    end
+
+    it do
+      expect(client_double).to have_received(:persist_job).with item
+      expect(item.finished_at).not_to be_nil
+    end
+  end
+
+  describe '#enqueue_outgoing_jobs' do
+    let(:outgoing) { ["A|#{SecureRandom.uuid}"] }
+    let(:client_double) do
+      double(
+        find_job: nil,
+        check_or_lock: nil,
+        release_lock: nil
+      )
+    end
+    let(:a_item) do
+      described_class.new(
+        workflow_id: SecureRandom.uuid,
+        id: SecureRandom.uuid,
+        started_at: started_at
+      )
+    end
+    before do
+      allow(Dwf::Client).to receive(:new).and_return client_double
+      allow(a_item).to receive(:persist_and_perform_async!)
+      allow(client_double)
+        .to receive(:find_job).and_return a_item
+      item.enqueue_outgoing_jobs
+    end
+
+    context 'outgoing jobs ready to start' do
+      let(:started_at) { nil }
+      it { expect(a_item).to have_received(:persist_and_perform_async!) }
+    end
+
+    context 'outgoing jobs havent ready to start' do
+      let(:started_at) { Time.now.to_i }
+      it { expect(a_item).not_to have_received(:persist_and_perform_async!) }
+    end
+  end
+end

data/spec/dwf/utils_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe Dwf::Utils, utils: true do
+  describe '#symbolize_keys' do
+    let(:expected) do
+      {
+        incoming: ['S'],
+        outgoing: %w[A B],
+        klass: 'H'
+      }
+    end
+
+    let(:hash) do
+      {
+        'incoming' => ['S'],
+        'outgoing' => %w[A B],
+        'klass' => 'H'
+      }
+    end
+
+    it { expect(described_class.symbolize_keys(hash)).to eq expected }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,101 @@
+require 'dwf'
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dwf
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - dthtien
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-04 00:00:00.000000000 Z
+date: 2021-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -73,12 +73,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/build_gem.yaml"
+- ".github/workflows/test.yaml"
 - ".gitignore"
+- ".rspec"
 - ".ruby-version"
 - Gemfile
 - LICENSE.txt
 - README.md
-- dwf-0.1.0.gem
 - dwf.gemspec
 - lib/dwf.rb
 - lib/dwf/callback.rb
@@ -87,11 +89,14 @@ files:
 - lib/dwf/utils.rb
 - lib/dwf/worker.rb
 - lib/dwf/workflow.rb
+- spec/dwf/item_spec.rb
+- spec/dwf/utils_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/dthtien/wf
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -107,8 +112,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.2.3
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gush cloned without ActiveJob but requried Sidekiq. This project is for researching
   DSL purpose
-test_files: []
+test_files:
+- spec/dwf/item_spec.rb
+- spec/dwf/utils_spec.rb
+- spec/spec_helper.rb