dwf 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d37bac690df59157001462d6017a8a63d3956a280689de789d9a99163a476832
-  data.tar.gz: 935d76e99ff5ca109f00be80c01950c20272969f556fc70d0e1ef5c026c024e4
+  metadata.gz: fba0bf3a5195787db6540ddfe2431a265b09b78859f333ed9295b4e4677d0a25
+  data.tar.gz: 66488a6107dec45933fc1631e713db3fb43a6e31dbd5d5a351a79a6bbb31f1f7
 SHA512:
-  metadata.gz: c105d178cd44f12577ee9eeed5065768a23adc508da7d686e50c46d26e4eed9c160dc71b10942ab8041e38f6c732384742f40fa9d3076e0a924fbdfb1cd62845
-  data.tar.gz: ad9c6f1041f092847364e91408887be311435be62e0502ab54f2b408c92fc598a879792eabb00c81b05d5c97fb37ea4a19136765d54b0d4efe9dbdb9ad2f64b5
+  metadata.gz: 88c01767c12ae11ff5f2aaeaf68b08a8c04a54bb909c51babd0f02cd915b1c8d790131c197034bbce46a8ab4d8e0dfa40a0f7afe4f1353bff65d952654438454
+  data.tar.gz: 799827548d718f4ae763c00c4519f636f448eb0aba40e6d5a60f83443cf2f479b3d8fbfd536744e11fc9092d22be3a18a7bb764b25cafcb27eccef3168ab0170

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 All notable changes to this project will be documented in this file.
+## 0.1.10
+### Added
+- Allow to use argument within workflow and update the defining callback way
+```
+class TestWf < Dwf::Workflow
+  def configure(arguments)
+    run A
+    run B, after: A, params: argument
+    run C, after: A, params: argument
+  end
+end
+
+wf = TestWf.create(arguments)
+wf.callback_type = Dwf::Workflow::SK_BATCH
+
+```
+- Support `find` workflow and `reload` workflow
+```
+wf = TestWf.create
+Dwf::Workflow.find(wf.id)
+wf.reload
+```
+
 ## 0.1.9
 ### Added
 ### Fixed

data/README.md

@@ -41,7 +41,8 @@ end
 
 ### Execute flow
 ```ruby
-wf = TestWf.create(callback_type: Dwf::Workflow::SK_BATCH)
+wf = TestWf.create
+wf.callback_type = Dwf::Workflow::SK_BATCH
 wf.start!
 ```
 
@@ -125,8 +126,11 @@ end
 - [x] Add github workflow
 - [x] Redis configurable
 - [x] Pinelining
-- [ ] [WIP] Test
+- [X] Test
+- [ ] Consistent item name
 - [ ] Support [Resque](https://github.com/resque/resque)
+- [ ] Key value store plugable
+  - [ ] research https://github.com/moneta-rb/moneta
 
 # References
 - https://github.com/chaps-io/gush

data/dwf.gemspec

@@ -6,7 +6,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "dwf"
-  spec.version       = '0.1.9'
+  spec.version       = '0.1.10'
   spec.authors       = ["dthtien"]
   spec.email         = ["tiendt2311@gmail.com"]
 

data/lib/dwf/client.rb

@@ -1,3 +1,5 @@
+require_relative 'errors'
+
 module Dwf
   class Client
     attr_reader :config
@@ -20,6 +22,16 @@ module Dwf
       Dwf::Item.from_hash(Dwf::Utils.symbolize_keys(data))
     end
 
+    def find_workflow(id)
+      data = redis.get("dwf.workflows.#{id}")
+      raise WorkflowNotFound, "Workflow with given id doesn't exist" if data.nil?
+
+      hash = JSON.parse(data)
+      hash = Dwf::Utils.symbolize_keys(hash)
+      nodes = parse_nodes(id)
+      workflow_from_hash(hash, nodes)
+    end
+
     def persist_job(job)
       redis.hset("dwf.jobs.#{job.workflow_id}.#{job.klass}", job.id, job.as_json)
     end
@@ -99,6 +111,28 @@ module Dwf
       job
     end
 
+    def parse_nodes(id)
+      keys = redis.scan_each(match: "dwf.jobs.#{id}.*")
+
+      keys.map do |key|
+        redis.hvals(key).map do |json|
+          Dwf::Utils.symbolize_keys JSON.parse(json)
+        end
+      end.flatten
+    end
+
+    def workflow_from_hash(hash, nodes = [])
+      flow = Module.const_get(hash[:klass]).new(*hash[:arguments])
+      flow.jobs = []
+      flow.stopped = hash.fetch(:stopped, false)
+      flow.id = hash[:id]
+      flow.jobs = nodes.map do |node|
+        Dwf::Item.from_hash(node)
+      end
+
+      flow
+    end
+
     def redis
       @redis ||= Redis.new(config.redis_opts)
     end

data/lib/dwf/errors.rb

@@ -0,0 +1,3 @@
+module Dwf
+    class WorkflowNotFound < StandardError; end
+end

data/lib/dwf/workflow.rb

@@ -10,8 +10,8 @@ module Dwf
       BUILD_IN = 'build-in',
       SK_BATCH = 'sk-batch'
     ].freeze
-    attr_reader :dependencies, :jobs, :started_at, :finished_at, :persisted, :stopped,
-                :callback_type
+    attr_accessor :jobs, :callback_type, :stopped, :id
+    attr_reader :dependencies, :started_at, :finished_at, :persisted, :arguments
 
     class << self
       def create(*args)
@@ -19,41 +19,58 @@ module Dwf
         flow.save
         flow
       end
+
+      def find(id)
+        Dwf::Client.new.find_workflow(id)
+      end
     end
 
-    def initialize(options = {})
+    def initialize(*args)
       @dependencies = []
-      @id = id
+      @id = build_id
       @jobs = []
       @persisted = false
       @stopped = false
-      @callback_type = options[:callback_type] || BUILD_IN
+      @arguments = args
+      @callback_type = BUILD_IN
 
       setup
     end
 
+    def persist!
+      client.persist_workflow(self)
+      jobs.each(&:persist!)
+      mark_as_persisted
+      true
+    end
+
+    alias save persist!
+
     def start!
+      mark_as_started
+      persist!
       initial_jobs.each do |job|
         cb_build_in? ? job.persist_and_perform_async! : Dwf::Callback.new.start(job)
       end
     end
 
-    def save
-      client.persist_workflow(self)
-      jobs.each(&:persist!)
-      mark_as_persisted
-      true
+    def reload
+      flow = self.class.find(id)
+      self.stopped = flow.stopped
+      self.jobs = flow.jobs
+
+      self
     end
 
     def cb_build_in?
       callback_type == BUILD_IN
     end
 
-    def id
-      @id ||= client.build_workflow_id
+    def build_id
+      client.build_workflow_id
     end
 
-    def configure; end
+    def configure(*arguments); end
 
     def run(klass, options = {})
       node = klass.new(
@@ -140,7 +157,6 @@ module Dwf
       @stopped = false
     end
 
-
     private
 
     def initial_jobs
@@ -148,7 +164,7 @@ module Dwf
     end
 
     def setup
-      configure
+      configure(*arguments)
       resolve_dependencies
     end
 

data/spec/dwf/client_spec.rb

@@ -39,6 +39,29 @@ describe Dwf::Client, client: true do
     end
   end
 
+  describe '#find_workflow' do
+    before do
+      wf = Dwf::Workflow.new
+      wf.id = workflow_id
+      wf.save
+      j = Dwf::Item.new(id: id, workflow_id: workflow_id)
+      j.persist!
+    end
+
+    it do
+      wf = client.find_workflow(workflow_id)
+
+      expect(wf).not_to be_nil
+      expect(wf.jobs.first).to be_kind_of(Dwf::Item)
+    end
+
+    it do
+      expect do
+        client.find_workflow(SecureRandom.uuid)
+      end.to raise_error Dwf::WorkflowNotFound
+    end
+  end
+
   describe '#persist_job' do
     let!(:job) { Dwf::Item.new(workflow_id: workflow_id, id: id) }
 

data/spec/dwf/workflow_spec.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'mock_redis'
+class AItem < Dwf::Item; end
+
+class BItem < Dwf::Item; end
+
+class CItem < Dwf::Item; end
+
+describe Dwf::Workflow, workflow: true do
+  let(:workflow_id) { SecureRandom.uuid }
+  let(:item_id) { SecureRandom.uuid }
+  let(:client) do
+    double(
+      persist_workflow: nil,
+      persist_job: nil,
+      build_workflow_id: workflow_id,
+      build_job_id: item_id,
+      find_workflow: nil
+    )
+  end
+  before do
+    allow(Dwf::Client).to receive(:new).and_return client
+  end
+
+  describe '#create' do
+    it do
+      workflow = described_class.create
+      expect(client).to have_received(:persist_workflow)
+        .with(an_instance_of(described_class))
+      expect(workflow.id).to eq workflow_id
+      expect(workflow.persisted).to be_truthy
+    end
+  end
+
+  describe '#find' do
+    before { Dwf::Workflow.find(workflow_id) }
+
+    it { expect(client).to have_received(:find_workflow).with(workflow_id) }
+  end
+
+  describe '#persist!' do
+    let(:workflow) { described_class.new }
+    let(:job) do
+      Dwf::Item.new(
+        worflow_id: workflow_id,
+        id: item_id
+      )
+    end
+    before do
+      workflow.jobs << job
+      workflow.persist!
+    end
+
+    it do
+      expect(client).to have_received(:persist_workflow)
+        .with(an_instance_of(described_class))
+      expect(client).to have_received(:persist_job).with(job)
+      expect(workflow.id).to eq workflow_id
+      expect(workflow.persisted).to be_truthy
+    end
+  end
+
+  describe '#start' do
+    let(:workflow) { described_class.new }
+    let(:job) do
+      Dwf::Item.new(
+        worflow_id: workflow_id,
+        id: item_id
+      )
+    end
+    before do
+      workflow.jobs << job
+      workflow.persist!
+    end
+
+    it do
+      expect(client).to have_received(:persist_workflow)
+        .with(an_instance_of(described_class))
+      expect(client).to have_received(:persist_job).with(job)
+      expect(workflow.id).to eq workflow_id
+      expect(workflow.persisted).to be_truthy
+      expect(workflow.stopped).to be_falsy
+    end
+  end
+
+  describe '#run' do
+    let!(:workflow) { described_class.new }
+
+    before do
+      workflow.run AItem, after: BItem, before: CItem
+    end
+
+    it do
+      expect(workflow.jobs).not_to be_empty
+      expected = [
+        {
+          from: BItem.to_s,
+          to: "AItem|#{item_id}"
+        },
+        {
+          from: "AItem|#{item_id}",
+          to: CItem.to_s
+        }
+      ]
+      expect(workflow.dependencies).to match_array expected
+    end
+  end
+
+  describe '#find_job' do
+    let!(:workflow) { described_class.new }
+    before do
+      workflow.jobs = [
+        AItem.new,
+        BItem.new(id: item_id),
+        CItem.new
+      ]
+    end
+
+    it 'searches by klass' do
+      job = workflow.find_job('AItem')
+      expect(job).to be_kind_of AItem
+    end
+
+    it 'searches by name' do
+      job = workflow.find_job("BItem|#{item_id}")
+      expect(job).to be_kind_of BItem
+    end
+  end
+
+  describe '#setup' do
+    let!(:workflow) { described_class.new }
+    before do
+      workflow.run AItem
+      workflow.run BItem, after: AItem
+      workflow.run CItem, after: BItem
+
+      workflow.send(:setup)
+    end
+
+    it do
+      job_a = workflow.find_job('AItem')
+
+      expect(job_a.incoming).to be_empty
+      expect(job_a.outgoing).to eq ["BItem|#{item_id}"]
+
+      job_b = workflow.find_job('BItem')
+
+      expect(job_b.incoming).to eq ['AItem']
+      expect(job_b.outgoing).to eq ["CItem|#{item_id}"]
+
+      job_c = workflow.find_job('CItem')
+
+      expect(job_c.incoming).to eq ['BItem']
+      expect(job_c.outgoing).to be_empty
+    end
+  end
+
+  describe '#callback_type' do
+    let!(:workflow) { described_class.new }
+
+    it {
+      expect(workflow.callback_type).to eq described_class::BUILD_IN
+      workflow.callback_type = described_class::SK_BATCH
+      expect(workflow.callback_type).to eq described_class::SK_BATCH
+    }
+  end
+
+  describe '#reload' do
+    let!(:workflow) do
+      flow = described_class.new
+      flow.id = workflow_id
+      flow
+    end
+
+    before do
+      allow(client).to receive(:find_workflow).and_return workflow
+      workflow.reload
+    end
+
+    it { expect(client).to have_received(:find_workflow).with(workflow_id) }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dwf
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - dthtien
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-10 00:00:00.000000000 Z
+date: 2021-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -101,6 +101,7 @@ files:
 - lib/dwf/callback.rb
 - lib/dwf/client.rb
 - lib/dwf/configuration.rb
+- lib/dwf/errors.rb
 - lib/dwf/item.rb
 - lib/dwf/utils.rb
 - lib/dwf/version.rb
@@ -111,6 +112,7 @@ files:
 - spec/dwf/item_spec.rb
 - spec/dwf/utils_spec.rb
 - spec/dwf/worker_spec.rb
+- spec/dwf/workflow_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/dthtien/wf
 licenses:
@@ -142,4 +144,5 @@ test_files:
 - spec/dwf/item_spec.rb
 - spec/dwf/utils_spec.rb
 - spec/dwf/worker_spec.rb
+- spec/dwf/workflow_spec.rb
 - spec/spec_helper.rb