pallets 0.1.0

data/lib/pallets/serializers/base.rb

@@ -0,0 +1,13 @@
+module Pallets
+  module Serializers
+    class Base
+      def dump(data)
+        raise NotImplementedError
+      end
+
+      def load(data)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/pallets/serializers/json.rb

@@ -0,0 +1,15 @@
+require 'json'
+
+module Pallets
+  module Serializers
+    class Json
+      def dump(data)
+        JSON.generate(data)
+      end
+
+      def load(data)
+        JSON.parse(data)
+      end
+    end
+  end
+end

data/lib/pallets/task.rb

@@ -0,0 +1,12 @@
+module Pallets
+  class Task
+    attr_reader :context
+
+    def initialize(context = {})
+      @context = context
+    end
+
+    def run
+    end
+  end
+end

data/lib/pallets/version.rb

@@ -0,0 +1,3 @@
+module Pallets
+  VERSION = "0.1.0"
+end

data/lib/pallets/worker.rb

@@ -0,0 +1,109 @@
+module Pallets
+  class Worker
+    attr_reader :manager
+
+    def initialize(manager)
+      @manager = manager
+      @current_job = nil
+      @needs_to_stop = false
+      @thread = nil
+    end
+
+    def start
+      @thread ||= Thread.new { work }
+    end
+
+    def graceful_shutdown
+      @needs_to_stop = true
+    end
+
+    def hard_shutdown
+      return unless @thread
+      @thread.raise Pallets::Shutdown
+    end
+
+    def needs_to_stop?
+      @needs_to_stop
+    end
+
+    def id
+      "W#{@thread.object_id.to_s(36)}".upcase if @thread
+    end
+
+    private
+
+    def work
+      loop do
+        break if needs_to_stop?
+
+        @current_job = backend.pick
+        # No need to requeue because of the reliability queue
+        break if needs_to_stop?
+        next if @current_job.nil?
+
+        process @current_job
+
+        @current_job = nil
+      end
+      @manager.remove_worker(self)
+    rescue Pallets::Shutdown
+      @manager.remove_worker(self)
+    rescue => ex
+      @manager.replace_worker(self)
+    end
+
+    def process(job)
+      Pallets.logger.info "[#{id}] Picked job: #{job}"
+      begin
+        job_hash = serializer.load(job)
+      rescue
+        # We ensure only valid jobs are created. If something fishy reaches this
+        # point, just discard it
+        backend.discard(job)
+        return
+      end
+
+      task_class = job_hash["class_name"].constantize
+      task = task_class.new(job_hash["context"])
+      begin
+        task.run
+      rescue => ex
+        handle_job_error(ex, job, job_hash)
+      else
+        backend.save(job_hash["workflow_id"], job)
+        Pallets.logger.info "[#{id}] Successfully processed #{job}"
+      end
+    end
+
+    def handle_job_error(ex, job, job_hash)
+      Pallets.logger.error "[#{id}] Error while processing: #{ex}"
+      failures = job_hash.fetch('failures', 0) + 1
+      new_job = serializer.dump(job_hash.merge(
+        'failures' => failures,
+        'failed_at' => Time.now.to_f,
+        'error_class' => ex.class.name,
+        'error_message' => ex.message
+      ))
+      if failures < job_hash['max_failures']
+        retry_at = Time.now.to_f + backoff_in_seconds(failures)
+        backend.retry(new_job, job, retry_at)
+        Pallets.logger.info "[#{id}] Scheduled job for retry"
+      else
+        backend.give_up(new_job, job, Time.now.to_f)
+        Pallets.logger.info "[#{id}] Given up on job"
+      end
+    end
+
+    def backoff_in_seconds(count)
+      count ** 4 + 6
+    end
+
+    def backend
+      @backend ||= Pallets.backend
+    end
+
+    def serializer
+      @serializer ||= Pallets.serializer
+    end
+  end
+end

data/lib/pallets/workflow.rb

@@ -0,0 +1,58 @@
+module Pallets
+  class Workflow
+    extend DSL::Workflow
+
+    attr_reader :context
+
+    def initialize(context = {})
+      @id = nil
+      @context = context
+    end
+
+    def run
+      backend.run_workflow(id, jobs_with_order)
+      id
+    end
+
+    def id
+      @id ||= begin
+        initials = self.class.name.gsub(/[^A-Z]+([A-Z])/, '\1')[0,3]
+        random = SecureRandom.hex(5)
+        "P#{initials}#{random}".upcase
+      end
+    end
+
+    private
+
+    def jobs_with_order
+      self.class.graph.sorted_with_order.map do |task_name, order|
+        job = serializer.dump(job_hash.merge(self.class.task_config[task_name]))
+        [order, job]
+      end
+    end
+
+    def job_hash
+      {
+        'workflow_id' => id,
+        'context'     => context,
+        'created_at'  => Time.now.to_f
+      }
+    end
+
+    def backend
+      Pallets.backend
+    end
+
+    def serializer
+      Pallets.serializer
+    end
+
+    def self.task_config
+      @task_config ||= {}
+    end
+
+    def self.graph
+      @graph ||= Graph.new
+    end
+  end
+end

data/pallets.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pallets/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "pallets"
+  spec.version       = Pallets::VERSION
+  spec.authors       = ["Andrei Horak"]
+  spec.email         = ["linkyndy@gmail.com"]
+
+  spec.summary       = 'Toy workflow engine, written in Ruby'
+  spec.description   = 'Toy workflow engine, written in Ruby'
+  spec.homepage      = 'https://github.com/linkyndy/pallets'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = ["pallets"]
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport"
+  spec.add_dependency "redis"
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "timecop"
+  spec.add_development_dependency "fuubar"
+end

metadata

@@ -0,0 +1,177 @@
+--- !ruby/object:Gem::Specification
+name: pallets
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrei Horak
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-09-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fuubar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Toy workflow engine, written in Ruby
+email:
+- linkyndy@gmail.com
+executables:
+- pallets
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CONTRIBUTING.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/pallets
+- bin/setup
+- lib/pallets.rb
+- lib/pallets/backends/base.rb
+- lib/pallets/backends/redis.rb
+- lib/pallets/backends/scripts/discard.lua
+- lib/pallets/backends/scripts/give_up.lua
+- lib/pallets/backends/scripts/reschedule_all.lua
+- lib/pallets/backends/scripts/retry.lua
+- lib/pallets/backends/scripts/run_workflow.lua
+- lib/pallets/backends/scripts/save.lua
+- lib/pallets/cli.rb
+- lib/pallets/configuration.rb
+- lib/pallets/dsl/workflow.rb
+- lib/pallets/errors.rb
+- lib/pallets/graph.rb
+- lib/pallets/manager.rb
+- lib/pallets/pool.rb
+- lib/pallets/scheduler.rb
+- lib/pallets/serializers/base.rb
+- lib/pallets/serializers/json.rb
+- lib/pallets/task.rb
+- lib/pallets/version.rb
+- lib/pallets/worker.rb
+- lib/pallets/workflow.rb
+- pallets.gemspec
+homepage: https://github.com/linkyndy/pallets
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Toy workflow engine, written in Ruby
+test_files: []