sidekiq-workflow 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe0a36c7bd5d7ddf8d45311228f214d85776082a
-  data.tar.gz: d4313404aff1c62630a60c6a21a7e6fa0ae415ed
+  metadata.gz: c896e5dd3c9e174a591c2485eef0c2f8f993847f
+  data.tar.gz: d8d07782138963a6773a9ac6491316173a067aaa
 SHA512:
-  metadata.gz: 48b8abdbfb49c14a60a0d8ce19817a092c5558f51fe1ed86cc5ed74bd0b86ef4336e313d9f0ade78177c0a57f409c765757d7788e1cebc5f495f3bef988d29d5
-  data.tar.gz: ffb16abb61d2367583a06be9bc0d7341706e0e1dda1ecee7e74f55adbfb025ebd92bbf5d19218cc7ee298d39636658e768653949c4a1549baa705fbfd3ee4fb9
+  metadata.gz: ac3613923dca6a2881b4a75a3eae57adcaf118e88584af49fd1b24dacb7c7d991f2afc0d504396131cac1a03b76e4ebd32f8ca073d7ea7fead948171fa8470ee
+  data.tar.gz: 3a4f57e72957c252ff73f9f369caed3d9e53c11957769bd4c51b1bdfef498cd2c5fbe3115d975621c0e5d3d9e0896fbb6b21959cbaf503d347d1f3b3c8adb617

data/.gitignore

@@ -8,3 +8,4 @@
 /spec/reports/
 /tmp/
 *.gem
+.DS_Store

data/README.md

@@ -1,4 +1,7 @@
 # Sidekiq::Workflow
+_sidekiq-workflow is currently in development and is not yet suited to
+production use.  Please continue at your own risk...YOU HAVE BEEN
+WARNED!_
 
 ## Installation
 
@@ -18,7 +21,72 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+To run a workflow, sidekiq-workflow provides a simple interface:
+
+```
+Sidekiq::Workflow.run(sequence_definition)
+```
+
+A sequence definition is composed of an array-defined workflow.  The
+convention here is that jobs within the root array element are executed
+sequentially, one level deeper is executed in parallel.  Currently,
+sidekiq-workflow only supports simple embedded parallelism at one level.
+This means that it only supports workflows like the following:
+
+```
+         worker_2
+        /        \
+worker_1          worker_4
+        \        /
+         worker_3
+```
+A workflow like this is easily defined with the following structure:
+```ruby
+[
+  worker_1,
+  [worker_2, worker_3],
+  worker_4
+]
+```
+workers in the root array are processed sequentially and any workers in nested
+arrays are processes in parrallel.  The `worker_#` values above can be a
+strings or worker class names so:
+
+```ruby
+[
+  "AccountWorker",
+  ["UserWorker", "ProfileWorker"],
+  "EmailWorker"
+]
+```
+
+is equivalent to
+
+```ruby
+[
+  AccountWorker,
+  [UserWorker, ProfileWorker],
+  EmailWorker
+]
+```
+
+## Coming Soon
+
+Parameter passing and custom worker methods will be coming shortly as
+well as expanded workflow structure providing for any mixture of
+sequential and parallel definitions to make workflows like this
+possible:
+
+```
+                  worker_5
+                 /        \
+         worker_4          ----worker_7
+        /        \        /   /
+worker_1          worker_6   /
+        \                   /
+         worker_2---worker_3
+
+```
 
 ## Development
 
@@ -30,7 +98,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/sidekiq-workflow. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+Bug reports and pull requests are welcome on GitHub at https://github.com/michaelkelly322/sidekiq-workflow. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
 Please also take a look at [this post](https://badmonkeydev.wordpress.com/2015/05/23/code-review-best-practices/) for guidlines on submitting pull requests.  Thanks!
 

data/lib/sidekiq/workflow.rb

@@ -1,11 +1,49 @@
 require "sidekiq"
+require "redis-namespace"
+require "dm-core"
+require "dm-types"
+require "dm-redis-adapter"
 require "sidekiq/workflow/version"
+require "sidekiq/workflow/client_middleware"
+require "sidekiq/workflow/server_middleware"
+require "sidekiq/workflow/manager"
+require "sidekiq/workflow/node"
 
-module Sidekiq
-  module Workflow
-    def self.run(workflow_definition)
-      #WorkflowValidator.validate workflow_definition
-      #WorkflowManager.execute workflow_definition
+class TestWorker
+  include Sidekiq::Worker
+
+  def perform
+    sleep(5)
+  end
+end
+
+DataMapper.setup(:default, {:adapter  => "redis"})
+
+module Sidekiq::Workflow
+
+  def self.run(workflow_definition)
+    workflow = Workflow.create(workflow_definition)
+
+    Manager.execute workflow
+    workflow.id
+  end
+
+  class Workflow
+    include DataMapper::Resource
+
+    property :id, Serial
+    property :workflow_definition, Json
+    property :nodes, Json
+
+    def initialize(workflow_definition)
+      raise ArgumentError.new('workflow_definition can\'t be nil') unless workflow_definition
+      self.workflow_definition = workflow_definition
+      self.nodes = parsed_definition
+      binding.pry
+    end
+
+    def parsed_definition
+      @parsed_definition ||= Manager.parse workflow_definition
     end
   end
 end

data/lib/sidekiq/workflow/client_middleware.rb

@@ -0,0 +1,32 @@
+class Sidekiq::Workflow::ClientMiddleware
+  attr_accessor :wid, :nid
+
+  def call(worker_class, msg, queue, redis_pool)
+    puts "BEFORE:= Sidekiq::Workflow::ClientMiddleware called with:\n  worker_class: #{worker_class}\n  msg: #{msg}\n  queue: #{queue}\n  redis_pool: #{redis_pool}"
+    if is_sw_job? msg['args']
+      parse(msg['args']).tap do |params|
+        msg['sw_wid'] = params[1]
+        msg['sw_nid'] = params[2]
+      end
+    end
+    puts "AFTER:= Sidekiq::Workflow::ClientMiddleware called with:\n  worker_class: #{worker_class}\n  msg: #{msg}\n  queue: #{queue}\n  redis_pool: #{redis_pool}"
+    yield
+  end
+
+  protected
+
+  def is_sw_job?(args)
+    args.is_a?(Array) &&
+      args.last.is_a?(Array) &&
+      args.last.include?('sidekiq-workflow')
+  end
+
+  def parse(args)
+    # This also strips the sidekiq-workflow params from msg['args']
+    @args ||= args.pop
+  end
+
+  def set_queued_ids(jid)
+    Workflow.find(wid)
+  end
+end

data/lib/sidekiq/workflow/manager.rb

@@ -0,0 +1,41 @@
+module Sidekiq
+  module Workflow
+    class Manager
+      attr_accessor :definition
+
+      def self.parse(workflow_definition)
+        [].tap do |h|
+          workflow_definition.each_with_index do |node, i|
+            if node.is_a? Array
+              h[i] = self.parse(node)
+            else
+              h[i] = Node.create({
+                name: node,
+                description: '',
+                params: [],
+                queued_ids: [],
+                completed_ids: []
+              }).id
+
+            end
+
+          end
+        end
+      end
+
+      def self.execute(workflow)
+        n = workflow.nodes.first
+
+        if n.is_a? Array
+          n.map do |id|
+            node = Node.get(id)
+            class_eval "#{node.name}.perform_async(['sidekiq-workflow', #{workflow.id}, #{node.id}])"
+          end
+        else
+          node = Node.get(n)
+          class_eval "#{node.name}.perform_async(['sidekiq-workflow', #{workflow.id}, #{node.id}])"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/workflow/monitor.rb

@@ -0,0 +1,11 @@
+class Sidekiq::Workflow::Monitor
+  attr_accessor :wid
+  def self.[](wid)
+    self.new(wid)
+  end
+
+  def initialize(wid)
+    self.wid = wid
+  end
+
+end

data/lib/sidekiq/workflow/node.rb

@@ -0,0 +1,14 @@
+class Sidekiq::Workflow::Node
+  include DataMapper::Resource
+
+  property :id, Serial
+  property :name, String
+  property :description, String
+
+  property :params, Json
+  property :completed_ids, Json
+  property :queued_ids, Json
+
+end
+
+DataMapper.finalize

data/lib/sidekiq/workflow/server_middleware.rb

@@ -0,0 +1,23 @@
+module Sidekiq
+  module Workflow
+
+    class ServerMiddleware
+      def call(worker_class, msg, queue)
+        puts "Sidekiq::Workflow::ServerMiddleware called with:\n  worker_class: #{worker_class}\n  msg: #{msg}\n  queue: #{queue}"
+
+        yield
+        # find wid, find nid within the workflow, then move jid from queued_ids to completed_ids
+        if msg['sw_wid'] && msg['sw_nid']
+          workflow = Sidekiq::Workflow::Workflow.get(msg['sw_wid'])
+          nodes = workflow.nodes.map { |n| Node.get(n) }
+
+          workflow.nodes.shift
+          if workflow.save
+            Manager.execute(workflow) unless workflow.nodes.empty?
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq/workflow/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Workflow
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

data/sidekiq-workflow.gemspec

@@ -19,9 +19,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.required_ruby_version = '1.9'
+  spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2"
+  spec.add_development_dependency "pry"
 
   spec.add_runtime_dependency "sidekiq", "~> 3.3"
+  spec.add_runtime_dependency "dm-redis-adapter"
+  spec.add_runtime_dependency "dm-types"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-workflow
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Michael Kelly
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-06-09 00:00:00.000000000 Z
+date: 2015-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: dm-redis-adapter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dm-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Allows the execution of mixed parallel and sequential workflows.  A workflow
   can be defined at runtime and executed with full sidekiq integration.
 email:
@@ -84,8 +126,14 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- dump.rdb
 - lib/sidekiq/workflow.rb
+- lib/sidekiq/workflow/client_middleware.rb
+- lib/sidekiq/workflow/manager.rb
+- lib/sidekiq/workflow/monitor.rb
+- lib/sidekiq/workflow/node.rb
 - lib/sidekiq/workflow/railtie.rb
+- lib/sidekiq/workflow/server_middleware.rb
 - lib/sidekiq/workflow/version.rb
 - sidekiq-workflow.gemspec
 homepage: https://github.com/michaelkelly322/sidekiq-workflow
@@ -98,9 +146,9 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -108,7 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.7
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Complex workflow management in Sidekiq