dwf 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a08560cdfefa1c9f8e1f42416599e94550de0f0e1492bb5b05424e2bf94a482f
-  data.tar.gz: 84a83b6849b64e7fe5da74f5734e2d8faa96df25eef63a5d0de78229fcc733ba
+  metadata.gz: fb0f6f6f30b7ed1ba19120a4a1f77e450940e45433153de51f47df9f108d0fae
+  data.tar.gz: e0e86fdd169b7cbcdc811cf62f87b3043fbbd708bdf0f41fb006b392e4ab784b
 SHA512:
-  metadata.gz: 6eb847d86872dfa70c1a602a3ae8ef0e0ca166cfa22d0a5e3706454ef0144ac434b48489a916d811e9dc66ebb9ef4979b00b63dee8da4d08c44af6d1bd714dad
-  data.tar.gz: cda011bc87919531f5de78721c3640c2460119cb4dd5668bd4ebb8093e19d46e848d27cfc2112c10021bd41e52e2365ced092f779a25a1509086fdf7095221a9
+  metadata.gz: 4a3770d4d7f63b7ede02d59e1ce43cb2634b39cbd9e037c920e4f816acb7b393de0f9030fda735e8ea0873f3679da120ff405f725e87c60c1b9eb6a7fbf355e3
+  data.tar.gz: 469e25cdf11441c59c80ff47197dcabcf40e0b086ab06babb6b39a0597483517c479ee9ea65767819b69afebf83db6dcdb5926a9839a2e40d02c03cf39a6eb57

data/.github/workflows/build_gem.yaml

@@ -8,10 +8,6 @@ on:
     branches: [ master ]
     paths:
       - 'dwf.gemspec'
-  pull_request:
-    branches: [ master ]
-    paths:
-      - 'dwf.gemspec'
 
 jobs:
   build:

data/.github/workflows/test.yaml

@@ -3,7 +3,7 @@ name: Test
 on:
   push:
     branches:
-      - '**'
+      - 'master'
   pull_request:
     branches:
       - '**'

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 All notable changes to this project will be documented in this file.
+## 0.1.7
+### Added
+- Allow to config redis and queue
+
+```ruby
+Dwf.config do |config|
+  config.opts = { url 'redis://127.0.0.1:6379' }
+  config.namespace = 'dwf'
+end
+```
+
 ## 0.1.6
 ### Added
 - Sidekiq batch callback: separate batches

data/README.md

@@ -4,7 +4,7 @@
 # Installation
 ## 1. Add `dwf` to Gemfile
 ```ruby
-gem 'dwf', '~> 0.1.5'
+gem 'dwf', '~> 0.1.6'
 ```
 ## 2. Execute flow
 ### Declare jobs
@@ -38,12 +38,6 @@ class TestWf < Dwf::Workflow
 end
 ```
 
-#### Note
-`dwf` supports 2 callback types `Dwf::Workflow::BUILD_IN` and `Dwf::Workflow::SK_BATCH`
-- `Dwf::Workflow::BUILD_IN` is a build-in callback
-- `Dwf::Workflow::SK_BATCH` is [sidekiq batch](https://github.com/mperham/sidekiq/wiki/Batches) callback which required [`sidekiq-pro`](https://sidekiq.org/products/pro.html)
-
-By default `dwf` will use `Dwf::Workflow::BUILD_IN` callback.
 
 ### Execute flow
 ```ruby
@@ -51,6 +45,13 @@ wf = TestWf.create(callback_type: Dwf::Workflow::SK_BATCH)
 wf.start!
 ```
 
+#### Note
+`dwf` supports 2 callback types `Dwf::Workflow::BUILD_IN` and `Dwf::Workflow::SK_BATCH`
+- `Dwf::Workflow::BUILD_IN` is a build-in callback
+- `Dwf::Workflow::SK_BATCH` is [sidekiq batch](https://github.com/mperham/sidekiq/wiki/Batches) callback which required [`sidekiq-pro`](https://sidekiq.org/products/pro.html)
+
+By default `dwf` will use `Dwf::Workflow::BUILD_IN` callback.
+
 ### Output
 ```
 A Working
@@ -70,11 +71,25 @@ D say hello
 D Finished
 ```
 
+# Config redis and default queue
+```ruby
+Dwf.config do |config|
+  SENTINELS = [
+    { host: "127.0.0.1", port: 26380 },
+    { host: "127.0.0.1", port: 26381 }
+  ]
+  config.opts = { host: 'mymaster', sentinels: SENTINELS, role: :master }
+  config.namespace = 'dwf'
+end
+```
+
+
 # Todo
 - [x] Make it work
 - [x] Support pass params
 - [x] Support with build-in callback
 - [x] Add github workflow
+- [x] Redis configurable
 - [ ] [WIP] Test
 - [ ] Transfer output through each node
 - [ ] Support [Resque](https://github.com/resque/resque)

data/dwf.gemspec

@@ -3,10 +3,11 @@
 
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require_relative 'lib/dwf/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "dwf"
-  spec.version       = '0.1.6'
+  spec.version       = Dwf::VERSION
   spec.authors       = ["dthtien"]
   spec.email         = ["tiendt2311@gmail.com"]
 
@@ -27,5 +28,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'byebug', '~> 11.1.3'
   spec.add_dependency 'redis', '~> 4.2.0'
   spec.add_development_dependency 'rspec', '~> 3.2'
+  spec.add_development_dependency 'mock_redis', '~> 0.27.2'
   spec.add_dependency 'sidekiq', '~> 6.2.0'
 end

data/lib/dwf/callback.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
 require_relative 'client'
 
 module Dwf
   class Callback
+    DEFAULT_KEY = 'default_key'
+
     def process_next_step(status, options)
       previous_job_names = options['names']
       workflow_id = options['workflow_id']
@@ -12,7 +15,7 @@ module Dwf
       return if processing_job_names.empty?
 
       overall = Sidekiq::Batch.new(status.parent_bid)
-      overall.jobs { setup_batch(processing_job_names, workflow_id) }
+      overall.jobs { setup_batches(processing_job_names, workflow_id) }
     end
 
     def start(job)
@@ -21,38 +24,36 @@ module Dwf
 
     private
 
-    def setup_batch(processing_job_names, workflow_id)
+    def setup_batches(processing_job_names, workflow_id)
       jobs = fetch_jobs(processing_job_names, workflow_id)
       jobs_classification = classify_jobs jobs
 
-      jobs_classification.values.each do |batch_jobs|
-        batch = Sidekiq::Batch.new
-        batch.on(
-          :success,
-          'Dwf::Callback#process_next_step',
-          names: batch_jobs.map(&:klass),
-          workflow_id: workflow_id
-        )
-
-        batch.jobs do
-          batch_jobs.each do |job|
-            job.persist_and_perform_async! if job.ready_to_start?
-          end
+      jobs_classification.each do |key, batch_jobs|
+        with_lock workflow_id, key do
+          setup_batch(batch_jobs, workflow_id)
         end
       end
     end
 
+    def setup_batch(jobs, workflow_id)
+      batch = Sidekiq::Batch.new
+      batch.on(
+        :success,
+        'Dwf::Callback#process_next_step',
+        names: jobs.map(&:klass),
+        workflow_id: workflow_id
+      )
+      batch.jobs do
+        jobs.each { |job| job.persist_and_perform_async! if job.ready_to_start? }
+      end
+    end
+
     def classify_jobs(jobs)
       hash = {}
       jobs.each do |job|
         outgoing_jobs = job.outgoing
-        key = outgoing_jobs.empty? ? 'default_key' : outgoing_jobs.join
-
-        if hash[key].nil?
-          hash[key] = [job]
-        else
-          hash[key] = hash[key].push(job)
-        end
+        key = outgoing_jobs.empty? ? DEFAULT_KEY : outgoing_jobs.join
+        hash[key] = hash[key].nil? ? [job] : hash[key].push(job)
       end
 
       hash
@@ -64,13 +65,6 @@ module Dwf
       end.compact
     end
 
-    def perform_job(job_name, workflow_id)
-      with_lock workflow_id, job_name do
-        job = client.find_job(workflow_id, job_name)
-        job.persist_and_perform_async! if job.ready_to_start?
-      end
-    end
-
     def with_lock(workflow_id, job_name)
       client.check_or_lock(workflow_id, job_name)
       yield

data/lib/dwf/client.rb

@@ -1,5 +1,11 @@
 module Dwf
   class Client
+    attr_reader :config
+
+    def initialize(config = Dwf.configuration)
+      @config = config
+    end
+
     def find_job(workflow_id, job_name)
       job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_name)
       data = if job_name_match
@@ -94,7 +100,7 @@ module Dwf
     end
 
     def redis
-      @redis ||= Redis.new
+      @redis ||= Redis.new(config.redis_opts)
     end
   end
 end

data/lib/dwf/configuration.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Dwf
+  class Configuration
+    NAMESPACE = 'dwf'
+    REDIS_OPTS = { url: 'redis://localhost:6379' }.freeze
+
+    attr_accessor :redis_opts, :namespace
+
+    def initialize(hash = {})
+      @namespace = hash.fetch(:namespace, NAMESPACE)
+      @redis_opts = hash.fetch(:redis_url, REDIS_OPTS)
+    end
+  end
+end

data/lib/dwf/item.rb

@@ -1,10 +1,9 @@
 # frozen_string_literal: true
+
 require_relative 'client'
 
 module Dwf
   class Item
-    DEFAULT_QUEUE = 'default'
-
     attr_reader :workflow_id, :id, :params, :queue, :klass, :started_at,
       :enqueued_at, :finished_at, :failed_at, :callback_type
     attr_accessor :incoming, :outgoing
@@ -13,7 +12,7 @@ module Dwf
       @workflow_id = options[:workflow_id]
       @id = options[:id]
       @params = options[:params]
-      @queue = options[:queue] || DEFAULT_QUEUE
+      @queue = options[:queue]
       @incoming = options[:incoming] || []
       @outgoing = options[:outgoing] || []
       @klass = options[:klass] || self.class
@@ -41,7 +40,8 @@ module Dwf
     end
 
     def perform_async
-      Dwf::Worker.set(queue: queue).perform_async(workflow_id, name)
+      Dwf::Worker.set(queue: queue || client.config.namespace)
+                 .perform_async(workflow_id, name)
     end
 
     def name

data/lib/dwf/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Dwf
+  VERSION = '0.1.7'
+end

data/lib/dwf/worker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sidekiq'
 require_relative 'client'
 

data/lib/dwf.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require "bundler/setup"
 
 require 'sidekiq'
@@ -12,8 +13,15 @@ require_relative 'dwf/item'
 require_relative 'dwf/client'
 require_relative 'dwf/worker'
 require_relative 'dwf/callback'
+require_relative 'dwf/configuration'
 
 module Dwf
-  VERSION = '0.1.6'
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.config
+    yield configuration
+  end
 end
 

data/spec/dwf/client_spec.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'mock_redis'
+
+describe Dwf::Client, client: true do
+  let(:client) { described_class.new }
+  let(:workflow_id) { SecureRandom.uuid }
+  let(:id) { SecureRandom.uuid }
+  let(:redis) { Redis.new }
+  before do
+    redis_instance = MockRedis.new
+    allow(Redis).to receive(:new).and_return redis_instance
+  end
+
+  describe '#find_job' do
+    let!(:job) do
+      j = Dwf::Item.new(workflow_id: workflow_id, id: id)
+      j.persist!
+      j
+    end
+
+    context 'find by item class name' do
+      it {
+        item = client.find_job(workflow_id, Dwf::Item.name)
+        expect(item.workflow_id).to eq workflow_id
+        expect(item.id).to eq id
+        expect(item.name).to eq job.name
+      }
+    end
+
+    context 'find by item name' do
+      it {
+        item = client.find_job(workflow_id, job.name)
+        expect(item.workflow_id).to eq workflow_id
+        expect(item.id).to eq id
+        expect(item.name).to eq job.name
+      }
+    end
+  end
+
+  describe '#persist_job' do
+    let!(:job) { Dwf::Item.new(workflow_id: workflow_id, id: id) }
+
+    it do
+      expect(redis.exists?("dwf.jobs.#{job.workflow_id}.#{job.klass}"))
+        .to be_falsy
+
+      client.persist_job(job)
+
+      expect(redis.exists?("dwf.jobs.#{job.workflow_id}.#{job.klass}"))
+        .to be_truthy
+    end
+  end
+
+  describe '#persist_workflow' do
+    let(:workflow) { Dwf::Workflow.new }
+
+    it do
+      expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_falsy
+      client.persist_workflow(workflow)
+      expect(redis.exists?("dwf.workflows.#{workflow.id}")).to be_truthy
+    end
+  end
+
+  describe '#check_or_lock' do
+    before do
+      allow_any_instance_of(described_class).to receive(:sleep)
+    end
+
+    context 'job is running' do
+      let(:job_name) { 'ahihi' }
+
+      before do
+        allow(client).to receive(:set)
+        redis.set("wf_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}", 'running')
+        client.check_or_lock(workflow_id, job_name)
+      end
+
+      it { expect(client).not_to have_received(:set) }
+    end
+
+    context 'job is not running' do
+      let(:job_name) { 'ahihi' }
+
+      before do
+        allow(redis).to receive(:set)
+        client.check_or_lock(workflow_id, job_name)
+      end
+
+      it do
+        expect(redis).to have_received(:set)
+          .with("wf_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}", 'running')
+      end
+    end
+  end
+
+  describe '#release_lock' do
+    before do
+      allow(redis).to receive(:del)
+      client.release_lock(workflow_id, 'ahihi')
+    end
+
+    it do
+      expect(redis).to have_received(:del)
+        .with("dwf_enqueue_outgoing_jobs_#{workflow_id}-ahihi")
+    end
+  end
+
+  describe '#build_job_id' do
+    before do
+      allow(redis).to receive(:hexists)
+      client.build_job_id(workflow_id, 'ahihi')
+    end
+
+    it { expect(redis).to have_received(:hexists) }
+  end
+
+  describe '#build_workflow_id' do
+    before do
+      allow(redis).to receive(:exists?)
+      client.build_workflow_id
+    end
+
+    it { expect(redis).to have_received(:exists?) }
+  end
+
+  describe '#key_exists?' do
+    before do
+      allow(redis).to receive(:exists?)
+      client.key_exists?('ahihi')
+    end
+
+    it { expect(redis).to have_received(:exists?).with('ahihi') }
+  end
+
+  describe '#set' do
+    before do
+      allow(redis).to receive(:set)
+      client.set('ahihi', 'a')
+    end
+
+    it { expect(redis).to have_received(:set).with('ahihi', 'a') }
+  end
+
+  describe '#delete' do
+    before do
+      allow(redis).to receive(:del)
+      client.delete('ahihi')
+    end
+
+    it { expect(redis).to have_received(:del).with('ahihi') }
+  end
+end

data/spec/dwf/configuration_spec.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Dwf::Configuration, configuration: true do
+  let(:configuration) { described_class.new }
+
+  specify do
+    expect(configuration.namespace).to eq described_class::NAMESPACE
+    expect(configuration.redis_opts).to eq described_class::REDIS_OPTS
+  end
+end

data/spec/dwf/item_spec.rb

@@ -16,7 +16,7 @@ describe Dwf::Item, item: true do
       params: {},
       incoming: incoming,
       outgoing: outgoing,
-      queue: Dwf::Item::DEFAULT_QUEUE,
+      queue: Dwf::Configuration::NAMESPACE,
       klass: 'Dwf::Item',
       started_at: started_at,
       finished_at: finished_at,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dwf
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - dthtien
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-06 00:00:00.000000000 Z
+date: 2021-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: mock_redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.27.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.27.2
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -86,10 +100,14 @@ files:
 - lib/dwf.rb
 - lib/dwf/callback.rb
 - lib/dwf/client.rb
+- lib/dwf/configuration.rb
 - lib/dwf/item.rb
 - lib/dwf/utils.rb
+- lib/dwf/version.rb
 - lib/dwf/worker.rb
 - lib/dwf/workflow.rb
+- spec/dwf/client_spec.rb
+- spec/dwf/configuration_spec.rb
 - spec/dwf/item_spec.rb
 - spec/dwf/utils_spec.rb
 - spec/spec_helper.rb
@@ -118,6 +136,8 @@ specification_version: 4
 summary: Gush cloned without ActiveJob but requried Sidekiq. This project is for researching
   DSL purpose
 test_files:
+- spec/dwf/client_spec.rb
+- spec/dwf/configuration_spec.rb
 - spec/dwf/item_spec.rb
 - spec/dwf/utils_spec.rb
 - spec/spec_helper.rb