dwf 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ad9df2cc13b2cf7cdca6b837d6b4b1b97e3784002afd8f756f4dfd85017aa749
+  data.tar.gz: '0960d1ebafa201f77b56e1adb99a7df1237f0b277474f8b20815bd1617204393'
+SHA512:
+  metadata.gz: 1ad226e5469a263d8d59214d4678ee890af48e1b20b1e6e630be9650b8a8e4b54c6b0ca96534615369f68055d9b69b3f65d3e8ee515bc118df51b22e734fd5aa
+  data.tar.gz: efc316c11eb6d8dfc6b38922195a9af98418a62bd5e77812620dbe7194aabdae404b0a985dacf50a104c82fb9eeb3c45ada5abe8c5b37838d6dd8e7361c4a206

data/.gitignore

@@ -0,0 +1,6 @@
+.byebug_history
+tmp/
+Gemfile.lock
+wf-*.gem
+:w
+:W

data/.ruby-version

@@ -0,0 +1 @@
+3.0.0

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+gemspec
+
+source "https://gems.contribsys.com/" do
+  gem 'sidekiq-pro'
+end
+

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 dthtien
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,60 @@
+# DSL playground
+[Gush](https://github.com/chaps-io/gush) cloned without [ActiveJob](https://guides.rubyonrails.org/active_job_basics.html) but requried [Sidekiq](https://github.com/mperham/sidekiq). This project is for researching DSL purpose
+
+# Execute flow
+## Declare jobs
+
+```ruby
+require_relative './wf/item'
+
+class A < Wf::Item
+  def perform
+    puts "#{self.class.name} Working"
+    sleep 2
+    puts "#{self.class.name} Finished"
+  end
+end
+
+class E < A; end
+class B < A; end
+class C < E; end
+class D < E; end
+```
+
+## Declare flow
+```ruby
+class TestWf < Wf::Workflow
+  def configure
+    run A
+    run B, after: A
+    run C, after: A
+    run E, after: [B, C]
+    run D, after: [E]
+  end
+end
+
+```
+
+### Execute flow
+```ruby
+wf = TestWf.create
+wf.start!
+```
+
+### Output
+```
+A Working
+A Finished
+B Working
+C Working
+B Finished
+C Finished
+E Working
+E Finished
+D Working
+D Finished
+```
+
+# References
+- https://github.com/chaps-io/gush
+- https://github.com/mperham/sidekiq

data/dwf.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "dwf"
+  spec.version       = '0.1.0'
+  spec.authors       = ["dthtien"]
+  spec.email         = ["tiendt2311@gmail.com"]
+
+  spec.summary       = 'Gush cloned without ActiveJob but requried Sidekiq. This project is for researching DSL purpose'
+  spec.description   = 'Workflow'
+  spec.homepage      = 'https://github.com/dthtien/wf'
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 2.4.0"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  # guide at: https://bundler.io/guides/creating_gem.html
+
+  spec.add_development_dependency 'byebug', '~> 11.1.3'
+  spec.add_dependency 'redis', '~> 4.2.0'
+  spec.add_development_dependency 'rspec', '~> 3.2'
+  spec.add_dependency 'sidekiq', '~> 6.2.0'
+end

data/lib/dwf/callback.rb

@@ -0,0 +1,67 @@
+require 'sidekiq-pro'
+require_relative 'client'
+
+module Dwf
+  class Callback
+    def process_next_step(status, options)
+      previous_job_names = options['names']
+      workflow_id = options['workflow_id']
+      processing_job_names = previous_job_names.map do |job_name|
+        job = client.find_job(workflow_id, job_name)
+        job.outgoing
+      end.flatten.uniq
+      return if processing_job_names.empty?
+
+      overall = Sidekiq::Batch.new(status.parent_bid)
+      overall.jobs { setup_batch(processing_job_names, workflow_id) }
+    end
+
+    def start(job)
+      job.outgoing.any? ? start_with_batch(job) : job.perform_async
+    end
+
+    private
+
+    def setup_batch(processing_job_names, workflow_id)
+      batch = Sidekiq::Batch.new
+      batch.on(
+        :success,
+        'Dwf::Callback#process_next_step',
+        names: processing_job_names,
+        workflow_id: workflow_id
+      )
+
+      batch.jobs do
+        processing_job_names.each { |job_name| perform_job(job_name, workflow_id) }
+      end
+    end
+
+    def perform_job(job_name, workflow_id)
+      with_lock workflow_id, job_name do
+        job = client.find_job(workflow_id, job_name)
+        job.persist_and_perform_async! if job.ready_to_start?
+      end
+    end
+
+    def with_lock(workflow_id, job_name)
+      client.check_or_lock(workflow_id, job_name)
+      yield
+      client.release_lock(workflow_id, job_name)
+    end
+
+    def start_with_batch(job)
+      batch = Sidekiq::Batch.new
+      batch.on(
+        :success,
+        'Dwf::Callback#process_next_step',
+        names: [job.name],
+        workflow_id: job.workflow_id
+      )
+      batch.jobs { job.perform_async }
+    end
+
+    def client
+      @client ||= Dwf::Client.new
+    end
+  end
+end

data/lib/dwf/client.rb

@@ -0,0 +1,100 @@
+module Dwf
+  class Client
+    def find_job(workflow_id, job_name)
+      job_name_match = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(job_name)
+      data = if job_name_match
+               find_job_by_klass_and_id(workflow_id, job_name)
+             else
+               find_job_by_klass(workflow_id, job_name)
+             end
+
+      return nil if data.nil?
+
+      data = JSON.parse(data)
+      Dwf::Item.from_hash(Dwf::Utils.symbolize_keys(data))
+    end
+
+    def persist_job(job)
+      redis.hset("dwf.jobs.#{job.workflow_id}.#{job.klass}", job.id, job.as_json)
+    end
+
+    def check_or_lock(workflow_id, job_name)
+      key = "wf_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}"
+
+      if key_exists?(key)
+        sleep 2
+      else
+        set(key, 'running')
+      end
+    end
+
+    def release_lock(workflow_id, job_name)
+      delete("dwf_enqueue_outgoing_jobs_#{workflow_id}-#{job_name}")
+    end
+
+    def persist_workflow(workflow)
+      redis.set("dwf.workflows.#{workflow.id}", workflow.as_json)
+    end
+
+    def build_job_id(workflow_id, job_klass)
+      jid = nil
+
+      loop do
+        jid = SecureRandom.uuid
+        available = !redis.hexists(
+          "dwf.jobs.#{workflow_id}.#{job_klass}",
+          jid
+        )
+
+        break if available
+      end
+
+      jid
+    end
+
+    def build_workflow_id
+      wid = nil
+      loop do
+        wid = SecureRandom.uuid
+        available = !redis.exists?("dwf.workflow.#{wid}")
+
+        break if available
+      end
+
+      wid
+    end
+
+    def key_exists?(key)
+      redis.exists?(key)
+    end
+
+    def set(key, value)
+      redis.set(key, value)
+    end
+
+    def delete(key)
+      redis.del(key)
+    end
+
+    private
+
+    def find_job_by_klass_and_id(workflow_id, job_name)
+      job_klass, job_id = job_name.split('|')
+
+      redis.hget("dwf.jobs.#{workflow_id}.#{job_klass}", job_id)
+    end
+
+    def find_job_by_klass(workflow_id, job_name)
+      _new_cursor, result = redis.hscan("dwf.jobs.#{workflow_id}.#{job_name}", 0, count: 1)
+      return nil if result.empty?
+
+      _job_id, job = *result[0]
+
+      job
+    end
+
+    def redis
+      @redis ||= Redis.new
+    end
+  end
+end

data/lib/dwf/item.rb

@@ -0,0 +1,153 @@
+require_relative 'client'
+
+module Dwf
+  class Item
+    attr_reader :workflow_id, :id, :params, :queue, :klass, :started_at,
+      :enqueued_at, :finished_at, :failed_at
+    attr_accessor :incomming, :outgoing
+
+    def initialize(options = {})
+      @workflow_id = options[:workflow_id]
+      @id = options[:id]
+      @params = options[:params]
+      @queue = options[:queue] || 'default'
+      @incomming = options[:incoming] || []
+      @outgoing = options[:outgoing] || []
+      @klass = options[:klass] || self.class
+      @finished_at = options[:finished_at]
+      @enqueued_at = options[:enqueued_at]
+      @started_at = options[:started_at]
+    end
+
+    def self.from_hash(hash)
+      Module.const_get(hash[:klass]).new(hash)
+    end
+
+    def persist_and_perform_async!
+      enqueue!
+      persist!
+      perform_async
+    end
+
+    def perform; end
+
+    def perform_async
+      Dwf::Worker.set(queue: queue).perform_async(workflow_id, name)
+    end
+
+    def name
+      @name ||= "#{klass}|#{id}"
+    end
+
+    def no_dependencies?
+      incomming.empty?
+    end
+
+    def parents_succeeded?
+      incomming.all? do |name|
+        client.find_job(workflow_id, name).succeeded?
+      end
+    end
+
+    def enqueue!
+      @enqueued_at = current_timestamp
+      @started_at = nil
+      @finished_at = nil
+      @failed_at = nil
+    end
+
+    def mark_as_started
+      start!
+      persist!
+    end
+
+    def mark_as_finished
+      finish!
+      persist!
+    end
+
+    def start!
+      @started_at = current_timestamp
+      @failed_at = nil
+    end
+
+    def finish!
+      @finished_at = current_timestamp
+    end
+
+    def fail!
+      @finished_at = @failed_at = current_timestamp
+    end
+
+    def enqueued?
+      !enqueued_at.nil?
+    end
+
+    def finished?
+      !finished_at.nil?
+    end
+
+    def failed?
+      !failed_at.nil?
+    end
+
+    def succeeded?
+      finished? && !failed?
+    end
+
+    def started?
+      !started_at.nil?
+    end
+
+    def running?
+      started? && !finished?
+    end
+
+    def ready_to_start?
+      !running? && !enqueued? && !finished? && !failed? && parents_succeeded?
+    end
+
+    def current_timestamp
+      Time.now.to_i
+    end
+
+    def enqueue_outgoing_jobs
+      outgoing.each do |job_name|
+        client.check_or_lock(workflow_id, job_name)
+        out = client.find_job(workflow_id, job_name)
+        out.persist_and_perform_async! if out.ready_to_start?
+        client.release_lock(workflow_id, job_name)
+      end
+    end
+
+    def to_hash
+      {
+        id: id,
+        klass: klass.to_s,
+        queue: queue,
+        incoming: incomming,
+        outgoing: outgoing,
+        finished_at: finished_at,
+        enqueued_at: enqueued_at,
+        started_at: started_at,
+        failed_at: failed_at,
+        params: params,
+        workflow_id: workflow_id
+      }
+    end
+
+    def as_json
+      to_hash.to_json
+    end
+
+    def persist!
+      client.persist_job(self)
+    end
+
+    private
+
+    def client
+      @client ||= Dwf::Client.new
+    end
+  end
+end

data/lib/dwf/utils.rb

@@ -0,0 +1,38 @@
+module Dwf
+  module Utils
+    def self.symbolize_keys(obj)
+      case obj
+      when Array
+        obj.inject([]) do |res, val|
+          res << case val
+          when Hash, Array
+            symbolize_keys(val)
+          else
+            val
+          end
+
+          res
+        end
+      when Hash
+        obj.inject({}) do |res, (key, val)|
+          nkey = case key
+                 when String
+                   key.to_sym
+                 else
+                   key
+                 end
+          nval = case val
+                 when Hash, Array
+                   symbolize_keys(val)
+                 else
+                   val
+                 end
+          res[nkey] = nval
+          res
+        end
+      else
+        obj
+      end
+    end
+  end
+end

data/lib/dwf/worker.rb

@@ -0,0 +1,24 @@
+require 'sidekiq'
+require_relative 'client'
+
+module Dwf
+  class Worker
+    include Sidekiq::Worker
+
+    def perform(workflow_id, job_name)
+      job = client.find_job(workflow_id, job_name)
+      return job.enqueue_outgoing_jobs if job. succeeded?
+
+      job.mark_as_started
+      job.perform
+      job.mark_as_finished
+      # job.enqueue_outgoing_jobs
+    end
+
+    private
+
+    def client
+      @client ||= Dwf::Client.new
+    end
+  end
+end

data/lib/dwf/workflow.rb

@@ -0,0 +1,169 @@
+require_relative 'client'
+require_relative 'worker'
+require_relative 'callback'
+
+module Dwf
+  class Workflow
+    attr_reader :dependencies, :jobs, :started_at, :finished_at, :persisted, :stopped
+
+    class << self
+      def create
+        flow = new
+        flow.save
+        flow
+      end
+    end
+
+    def initialize
+      @dependencies = []
+      @id = id
+      @jobs = []
+      @persisted = false
+      @stopped = false
+
+      setup
+    end
+
+    def start!
+      initial_jobs.each do |job|
+        Dwf::Callback.new.start(job)
+      end
+    end
+
+    def save
+      client.persist_workflow(self)
+      jobs.each(&:persist!)
+      mark_as_persisted
+      true
+    end
+
+    def id
+      @id ||= client.build_workflow_id
+    end
+
+    def configure; end
+
+    def run(klass, options = {})
+      node = klass.new(
+        workflow_id: id,
+        id: client.build_job_id(id, klass.to_s),
+        params: options.fetch(:params, {}),
+        queue: options[:queue],
+      )
+
+      jobs << node
+
+      build_dependencies_structure(node, options)
+      node.name
+    end
+
+    def find_job(name)
+      match_data = /(?<klass>\w*[^-])-(?<identifier>.*)/.match(name.to_s)
+
+      if match_data.nil?
+        job = jobs.find { |node| node.klass.to_s == name.to_s }
+      else
+        job = jobs.find { |node| node.name.to_s == name.to_s }
+      end
+
+      job
+    end
+
+    def to_hash
+      name = self.class.to_s
+      {
+        name: name,
+        id: id,
+        arguments: @arguments,
+        total: jobs.count,
+        finished: jobs.count(&:finished?),
+        klass: name,
+        status: status,
+        stopped: stopped,
+        started_at: started_at,
+        finished_at: finished_at
+      }
+    end
+
+    def as_json
+      to_hash.to_json
+    end
+
+    def finished?
+      jobs.all?(&:finished?)
+    end
+
+    def started?
+      !!started_at
+    end
+
+    def running?
+      started? && !finished?
+    end
+
+    def failed?
+      jobs.any?(&:failed?)
+    end
+
+    def stopped?
+      stopped
+    end
+
+    def status
+      return :failed if failed?
+      return :running if running?
+      return :finished if finished?
+      return :stopped if stopped?
+
+      :running
+    end
+
+    def mark_as_persisted
+      @persisted = true
+    end
+
+    def mark_as_started
+      @stopped = false
+    end
+
+
+    private
+
+    def initial_jobs
+      jobs.select(&:no_dependencies?)
+    end
+
+    def setup
+      configure
+      resolve_dependencies
+    end
+
+    def resolve_dependencies
+      @dependencies.each do |dependency|
+        from = find_job(dependency[:from])
+        to   = find_job(dependency[:to])
+
+        to.incomming << dependency[:from]
+        from.outgoing << dependency[:to]
+      end
+    end
+
+    def build_dependencies_structure(node, options)
+      deps_after = [*options[:after]]
+
+      deps_after.each do |dep|
+        @dependencies << { from: dep.to_s, to: node.name.to_s }
+      end
+
+      deps_before = [*options[:before]]
+
+      deps_before.each do |dep|
+        @dependencies << { from: node.name.to_s, to: dep.to_s }
+      end
+    end
+
+    def client
+      @client ||= Dwf::Client.new
+    end
+  end
+end

data/lib/dwf.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+require "bundler/setup"
+
+require 'sidekiq'
+require 'sidekiq-pro'
+require 'json'
+require 'byebug'
+require 'redis'
+
+require_relative 'dwf/utils'
+require_relative 'dwf/workflow'
+require_relative 'dwf/item'
+require_relative 'dwf/client'
+require_relative 'dwf/worker'
+require_relative 'dwf/callback'
+
+module Dwf
+  VERSION = '0.1.0'
+end
+

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: dwf
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- dthtien
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-09-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 11.1.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 11.1.3
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 6.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 6.2.0
+description: Workflow
+email:
+- tiendt2311@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- dwf.gemspec
+- lib/dwf.rb
+- lib/dwf/callback.rb
+- lib/dwf/client.rb
+- lib/dwf/item.rb
+- lib/dwf/utils.rb
+- lib/dwf/worker.rb
+- lib/dwf/workflow.rb
+homepage: https://github.com/dthtien/wf
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Gush cloned without ActiveJob but requried Sidekiq. This project is for researching
+  DSL purpose
+test_files: []