pallets 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fd28e870c825c99c372243b13d14574d3d6ec36b83bf031230938fe0121caeae
+  data.tar.gz: f9ad65469c6d58510c01ed22591e875c941a3609eaff4191b1156ee6215a1748
+SHA512:
+  metadata.gz: 33aaa7382095bda19a78c7baa5bfab9cc2246aa3b8b309706b54ca381f34f06df2e0196a04b110c6daee6b5720894ba12ddadd89e952b947014dbb0df85ed8df
+  data.tar.gz: 69a052b8663b578757bcda235a2d46159cc74871c84efe319cbb5da1aaba3992d5bbc84261e65e8b0cd51e95f3022a67dc35dc27e696f6b648febc2054fe0e3a

data/.gitignore

@@ -0,0 +1,36 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rspec

@@ -0,0 +1,2 @@
+--format Fuubar
+--color

data/.travis.yml

@@ -0,0 +1,12 @@
+---
+language: ruby
+services:
+  - redis-server
+cache: bundler
+rvm:
+  - 2.1.10
+  - 2.2.10
+  - 2.3.7
+  - 2.4.4
+  - 2.5.1
+script: bundle exec rspec

data/CONTRIBUTING.md

@@ -0,0 +1,19 @@
+# Contributing
+
+Any contribution is **highly** appreciated, no matter whether it is a small bug fix, feature implementation or typo in the docs.
+
+You can contribute in many ways:
+
+* report bugs;
+* fix bugs;
+* implement features;
+* write documentation;
+* simply tell us what you love about pallets or what you'd like to see in it!
+
+## Getting Started!
+
+**Fork** this repo, do your magic, `rspec` and submit a pull request on the **master** branch!
+
+> Issues, features, bugs and questions are tracked at https://github.com/linkyndy/pallets/issues
+
+> Documentation is found at https://github.com/linkyndy/pallets/wiki

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pallets.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Andrei Horak
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,91 @@
+# Pallets
+
+[![Build Status](https://travis-ci.com/linkyndy/pallets.svg?branch=master)](https://travis-ci.com/linkyndy/pallets)
+
+Toy workflow engine, written in Ruby
+
+## It is plain simple!
+
+```ruby
+# my_workflow.rb
+require 'pallets'
+
+class MyWorkflow < Pallets::Workflow
+  task :foo
+  task :bar => :foo
+  task :baz => :foo
+  task :qux => [:bar, :baz]
+end
+
+class Foo < Pallets::Task
+  def run
+    puts 'I love Pallets! <3'
+  end
+end
+# [other task definitions are ommited, for now]
+
+MyWorkflow.new.run
+```
+
+That's basically it! Curious for more? Read on!
+
+> Don't forget to run pallets, so it can process your tasks: `bundle exec pallets -r ./my_workflow`
+
+## Features
+
+* faaast!
+* reliable
+* retries failed tasks
+* Redis backend out of the box
+* JSON serializer out of the box
+* beautiful DSL
+* convention over configuration
+* thoroughly tested
+
+## Installation
+
+```
+# Gemfile
+gem 'pallets'
+
+# or
+
+gem install pallets
+```
+
+## Configuration
+
+```ruby
+Pallets.configure do |c|
+  # How many workers to process incoming jobs?
+  c.concurrency = 2
+
+  c.backend = :redis
+  c.serializer = :json
+
+  c.backend_args = { db: 1 }
+
+  # What's the maximum allowed time to process a job?
+  c.job_timeout = 1800
+  # How many times should a job be retried?
+  c.max_failures = 3
+end
+```
+
+For the complete set of options, see [pallets/configuration.rb](lib/pallets/configuration.rb)
+
+## Motivation
+
+The main reason for Pallet's existence was the need of a fast, simple and reliable workflow engine, one that is easily extensible with various backends and serializer, one that does not lose your data and one that is intelligent enough to concurrently schedule a workflow's tasks.
+
+## Status
+
+Pallets is under active development and it is not _yet_ production-ready.
+
+## How to contribute?
+
+Any contribution is **highly** appreciated! See [CONTRIBUTING.md](CONTRIBUTING.md) for more details.
+
+## License
+
+See [LICENSE](LICENSE)

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pallets"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/pallets

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'pallets'
+require 'pallets/cli'
+
+cli = Pallets::CLI.new
+cli.run

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/pallets.rb

@@ -0,0 +1,53 @@
+require "pallets/version"
+
+require 'pallets/backends/base'
+require 'pallets/backends/redis'
+require 'pallets/configuration'
+require 'pallets/dsl/workflow'
+require 'pallets/errors'
+require 'pallets/graph'
+require 'pallets/manager'
+require 'pallets/pool'
+require 'pallets/scheduler'
+require 'pallets/serializers/base'
+require 'pallets/serializers/json'
+require 'pallets/task'
+require 'pallets/worker'
+require 'pallets/workflow'
+
+require 'active_support/inflector'
+require 'logger'
+
+module Pallets
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield configuration
+  end
+
+  def self.backend
+    @backend ||= begin
+      cls = "Pallets::Backends::#{configuration.backend.capitalize}".constantize
+      cls.new(
+        namespace: configuration.namespace,
+        blocking_timeout: configuration.blocking_timeout,
+        job_timeout: configuration.job_timeout,
+        pool_size: configuration.pool_size,
+        **configuration.backend_args
+      )
+    end
+  end
+
+  def self.serializer
+    @serializer ||= begin
+      cls = "Pallets::Serializers::#{configuration.serializer.capitalize}".constantize
+      cls.new
+    end
+  end
+
+  def self.logger
+    @logger ||= Logger.new(STDOUT)
+  end
+end

data/lib/pallets/backends/base.rb

@@ -0,0 +1,38 @@
+module Pallets
+  module Backends
+    class Base
+      # Picks a job that is ready for processing
+      def pick
+        raise NotImplementedError
+      end
+
+      # Saves a job after successfully processing it
+      def save(workflow_id, job)
+        raise NotImplementedError
+      end
+
+      # Discard a malformed job
+      def discard(job)
+        raise NotImplementedError
+      end
+
+      # Schedule a failed job for retry
+      def retry(job, old_job, at)
+        raise NotImplementedError
+      end
+
+      # Give up job after repeteadly failing to process it
+      def give_up(job, old_job, at)
+        raise NotImplementedError
+      end
+
+      def reschedule_all(earlier_than)
+        raise NotImplementedError
+      end
+
+      def run_workflow(workflow_id, jobs_with_dependencies)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/pallets/backends/redis.rb

@@ -0,0 +1,109 @@
+require 'redis'
+
+module Pallets
+  module Backends
+    class Redis < Base
+      def initialize(namespace:, blocking_timeout:, job_timeout:, pool_size:, **options)
+        @namespace = namespace
+        @blocking_timeout = blocking_timeout
+        @job_timeout = job_timeout
+        @pool = Pallets::Pool.new(pool_size) { ::Redis.new(options) }
+
+        @queue_key = "#{namespace}:queue"
+        @reliability_queue_key = "#{namespace}:reliability-queue"
+        @reliability_set_key = "#{namespace}:reliability-set"
+        @retry_set_key = "#{namespace}:retry-set"
+        @fail_set_key = "#{namespace}:fail-set"
+        @workflow_key = "#{namespace}:workflows:%s"
+
+        register_scripts
+      end
+
+      def pick
+        job = @pool.execute do |client|
+          client.brpoplpush(@queue_key, @reliability_queue_key, timeout: @blocking_timeout)
+        end
+        if job
+          # We store the job's timeout so we know when to retry jobs that are
+          # still on the reliability queue. We do this separately since there is
+          # no other way to atomically BRPOPLPUSH from the main queue to a
+          # sorted set
+          @pool.execute do |client|
+            client.zadd(@reliability_set_key, Time.now.to_f + @job_timeout, job)
+          end
+        end
+        job
+      end
+
+      def save(workflow_id, job)
+        @pool.execute do |client|
+          client.eval(
+            @scripts['save'],
+            [@workflow_key % workflow_id, @queue_key, @reliability_queue_key, @reliability_set_key],
+            [job]
+          )
+        end
+      end
+
+      def discard(job)
+        @pool.execute do |client|
+          client.eval(
+            @scripts['discard'],
+            [@reliability_queue_key, @reliability_set_key],
+            [job]
+          )
+        end
+      end
+
+      def retry(job, old_job, at)
+        @pool.execute do |client|
+          client.eval(
+            @scripts['retry'],
+            [@retry_set_key, @reliability_queue_key, @reliability_set_key],
+            [at, job, old_job]
+          )
+        end
+      end
+
+      def give_up(job, old_job, at)
+        @pool.execute do |client|
+          client.eval(
+            @scripts['give_up'],
+            [@fail_set_key, @reliability_queue_key, @reliability_set_key],
+            [at, job, old_job]
+          )
+        end
+      end
+
+      def reschedule_all(earlier_than)
+        @pool.execute do |client|
+          client.eval(
+            @scripts['reschedule_all'],
+            [@reliability_set_key, @reliability_queue_key, @retry_set_key, @queue_key],
+            [earlier_than]
+          )
+        end
+      end
+
+      def run_workflow(workflow_id, jobs_with_order)
+        @pool.execute do |client|
+          client.eval(
+            @scripts['run_workflow'],
+            [@workflow_key % workflow_id, @queue_key],
+            jobs_with_order
+          )
+        end
+      end
+
+      private
+
+      def register_scripts
+        @scripts ||= Dir["#{__dir__}/scripts/*.lua"].map do |file|
+          name = File.basename(file, '.lua')
+          script = File.read(file)
+          [name, script]
+        end.to_h
+      end
+    end
+  end
+end